diff --git a/.github/actions/setup-build-env/action.yml b/.github/actions/setup-build-env/action.yml index 39f0b82e..67c53d71 100644 --- a/.github/actions/setup-build-env/action.yml +++ b/.github/actions/setup-build-env/action.yml @@ -62,9 +62,12 @@ runs: # An Arm64 host runs the AArch64 output natively, so mk/arm64.mk # turns USE_QEMU off and the build resolves the interpreter and libc # from the running system instead of from a cross sysroot. - if [ "$LINK_MODE" = dynamic ] && [ "$ARCHITECTURE" = arm64 ] && \ - [ "$(uname -m)" != aarch64 ]; then - packages+=(gcc-aarch64-linux-gnu) + # ARM64 encoder and startup ABI tests also need the cross tools in + # static builds on an x86 host, with the cross libc headers the + # startup test's preload shim includes; without recommends, apt + # installs the compiler alone. + if [ "$ARCHITECTURE" = arm64 ] && [ "$(uname -m)" != aarch64 ]; then + packages+=(gcc-aarch64-linux-gnu libc6-dev-arm64-cross) fi sudo apt-get update -q -y sudo apt-get install -q -y --no-install-recommends "${packages[@]}" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e5b22a57..787f7dbc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,7 +23,9 @@ jobs: host-x86: name: ${{ matrix.architecture }}/${{ matrix.link_mode }}/${{ matrix.binding_mode }} (${{ matrix.compiler }}) runs-on: ubuntu-24.04 - timeout-minutes: 30 + # The stage 2 suites run the self-hosted compiler under qemu for every + # case, and the cross-compiled lanes need more than half an hour of it. + timeout-minutes: 60 strategy: # One target failing says nothing about the others, and finding out # which of them share the failure is the point of the matrix. diff --git a/Makefile b/Makefile index 9fbc6972..94c0ea7a 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ -# -fwrapv is required, not probed: hashmap_hash_index() carries the FNV-1a +# -fwrapv is required, not probed: hashmap_hash() carries the FNV-1a # accumulator in a signed int, because shecc has no 'unsigned' to compile # itself with, and the multiply there overflows by design. -CFLAGS := -O -g \ +CFLAGS := -O2 -g \ -std=c99 -pedantic -fwrapv # Every -Wno- that used to sit here has been earned away rather than renewed: @@ -24,6 +24,18 @@ check_flag = $(shell $(CC) $(1) -S -o /dev/null -xc /dev/null 2>/dev/null; \ # targets have no use for, so skip it when nothing is being compiled. STYLE_GOALS := check-style check-newline check-comments check-format check-shell \ indent install-hooks uninstall-hooks check-hooks + +VIR_FRONTEND_FIXTURES := $(sort $(wildcard tests/vir-frontend-*.c)) +VIR_NATIVE_SHARED_FIXTURES := $(sort $(wildcard tests/vir-direct-global-*.c) \ + $(wildcard tests/vir-direct-branch-*.c) \ + $(wildcard tests/vir-direct-narrow-global.c) \ + tests/vir-direct-external-pointer-origin.c \ + tests/vir-direct-external-typed-pointer.c \ + tests/vir-direct-external-typed-pointer-offset.c) +VIR_NATIVE_X64_FIXTURES := $(sort $(wildcard tests/vir-direct-x64*.c) \ + $(VIR_NATIVE_SHARED_FIXTURES)) +VIR_NATIVE_ARM64_FIXTURES := $(sort $(wildcard tests/vir-direct-arm64*.c) \ + $(VIR_NATIVE_SHARED_FIXTURES)) ifneq ($(filter-out $(STYLE_GOALS),$(or $(MAKECMDGOALS),all)),) $(foreach flag, $(CFLAGS_TO_CHECK), $(eval CFLAGS += $(call check_flag, $(flag)))) endif @@ -80,7 +92,7 @@ endif TRANSLATION_DEFS = "\#define SHECC_TRANSLATION_DATE \"$(TRANSLATION_DATE)\"\n\#define SHECC_TRANSLATION_TIME \"$(TRANSLATION_TIME)\"\n" # --dump-ir is what makes out/shecc-stage1.log the IR of the stage 1 build # rather than an empty file. It is the only thing in the tree that exercises -# dump_insn()/dump_ph2_ir(), and it is what a failed CI run uploads. +# vir_print(), and it is what a failed CI run uploads. STAGE0_FLAGS ?= --dump-ir STAGE1_FLAGS ?= DYNLINK ?= 0 @@ -172,8 +184,93 @@ config: $(Q)$(CONFIG_CHECK_CMD) .PHONY: $(STYLE_GOALS) +.PHONY: check-vir-frontend-stage0 check-vir-frontend-stage2 \ + check-vir-stage0 check-vir-stage2 \ + check-vir-frontend-policy-stage0 check-vir-frontend-policy-stage2 \ + check-vir-opt-baseline + +check: check-stage0 check-stage2 check-abi-stage0 check-abi-stage2 \ + check-vir-core check-vir-frontend-stage0 check-vir-frontend-stage2 check-vir-fuzz check-vir-stage0 check-vir-stage2 \ + check-vir-multi-input-stage0 check-vir-multi-input-stage2 \ + check-vir-baseline check-vir-opt-baseline \ + check-vir-corpus-stage0 check-fuzz-stage0 check-vir-frontend-policy-stage0 + +# Random programs checked against the host compiler and built by shecc. +# FUZZ_SEEDS and FUZZ_START widen or move the run; a failing program is kept +# as out/fuzz-.c. +.PHONY: check-fuzz-stage0 +check-fuzz-stage0: $(OUT)/$(STAGE0) tests/fuzz.sh tests/fuzz-gen.c + TARGET_EXEC="$(TARGET_EXEC)" CC="$(CC)" bash tests/fuzz.sh 0 + +# Optimization must preserve each corpus program's behavior under --no-opt. +# Only stage 0 is part of check: under emulation, stage 2 compiles the corpus +# twice inside qemu, which costs more than the other checks. +.PHONY: check-vir-corpus-stage0 check-vir-corpus-stage2 +check-vir-corpus-stage0: $(OUT)/$(STAGE0) tests/vir-corpus.sh + TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-corpus.sh 0 $(ARCH) +check-vir-corpus-stage2: $(OUT)/$(STAGE2) tests/vir-corpus.sh + TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-corpus.sh 2 $(ARCH) + +.PHONY: check-vir-multi-input-stage0 check-vir-multi-input-stage2 +check-vir-multi-input-stage0: $(OUT)/$(STAGE0) tests/vir-multi-input.sh tests/vir-multi-input-root.c tests/vir-multi-input-main.c + TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-multi-input.sh 0 +check-vir-multi-input-stage2: $(OUT)/$(STAGE2) tests/vir-multi-input.sh tests/vir-multi-input-root.c tests/vir-multi-input-main.c + TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-multi-input.sh 2 + +ifeq ($(ARCH),x64) +.PHONY: check-vir-x64-encoders check-vir-x64-and-flags +check: check-vir-x64-encoders check-vir-x64-and-flags +check-vir-x64-encoders check-vir-x64-and-flags: check-vir-x64-%: $(OUT)/$(STAGE0) tests/vir-x64-%.c config $(wildcard src/*.c src/*.h) + $(CC) $(CFLAGS) tests/vir-x64-$*.c -o $(OUT)/vir-x64-$* + $(OUT)/vir-x64-$* + +.PHONY: check-vir-x64-fold +check: check-vir-x64-fold +check-vir-x64-fold: $(OUT)/$(STAGE0) tests/vir-x64-fold.c tests/vir-x64-liveout.c tests/vir-x64-liveout.sh tests/vir-x64-shift-counts.c config $(wildcard src/*.c src/*.h) + $(CC) $(CFLAGS) tests/vir-x64-fold.c -o $(OUT)/vir-x64-fold + $(OUT)/vir-x64-fold + CC="$(CC)" bash tests/vir-x64-liveout.sh $(OUT)/$(STAGE0) + +.PHONY: check-vir-native-x64-stage0 check-vir-native-x64-stage2 +check: check-vir-native-x64-stage0 check-vir-native-x64-stage2 +check-vir-native-x64-stage0: $(OUT)/$(STAGE0) $(VIR_NATIVE_X64_FIXTURES) tests/vir-direct-x64.sh tests/vir-direct-common.sh + bash tests/vir-direct-x64.sh 0 +check-vir-native-x64-stage2: $(OUT)/$(STAGE2) $(VIR_NATIVE_X64_FIXTURES) tests/vir-direct-x64.sh tests/vir-direct-common.sh + bash tests/vir-direct-x64.sh 2 + +# A foreign callee may leave garbage above a narrow return value, so the +# caller has to extend it. The callee is host assembly, preloaded at run time. +.PHONY: check-vir-call-return +check: check-vir-call-return +check-vir-call-return: $(OUT)/$(STAGE0) tests/vir-call-return.sh tests/vir-call-return.c + CC="$(CC)" bash tests/vir-call-return.sh $(OUT)/$(STAGE0) +endif -check: check-stage0 check-stage2 check-abi-stage0 check-abi-stage2 +ifeq ($(ARCH),arm64) +.PHONY: check-vir-native-arm64-stage0 check-vir-native-arm64-stage2 +check: check-vir-native-arm64-stage0 check-vir-native-arm64-stage2 +check: check-vir-arm64-branch-frame check-vir-arm64-startup-bank +.PHONY: check-vir-arm64-branch-frame check-vir-arm64-startup-bank +check: check-vir-arm64-select +check: check-vir-arm64-register-memory +.PHONY: check-vir-arm64-register-memory +check-vir-arm64-register-memory: $(OUT)/$(STAGE0) tests/arm64-register-memory-unit.c tests/arm64-register-memory.sh + CC="$(CC)" bash tests/arm64-register-memory.sh +.PHONY: check-vir-arm64-select +check-vir-arm64-select: $(OUT)/$(STAGE0) tests/vir-arm64-select-direct.c tests/vir-arm64-logical-gas.py + $(CC) $(CFLAGS) -Isrc tests/vir-arm64-select-direct.c -o $(OUT)/vir-arm64-select-direct + python3 tests/vir-arm64-logical-gas.py $(OUT)/vir-arm64-select-direct +check-vir-arm64-branch-frame: $(OUT)/$(STAGE0) tests/vir-arm64-branch-frame.c + $(CC) $(CFLAGS) -Isrc tests/vir-arm64-branch-frame.c -o $(OUT)/vir-arm64-branch-frame + $(OUT)/vir-arm64-branch-frame +check-vir-arm64-startup-bank: $(OUT)/$(STAGE0) tests/vir-arm64-startup-bank.sh + CC="$(CC)" A64_CC="$(if $(filter 1,$(USE_QEMU)),$(or $(ARCH_CC),aarch64-linux-gnu-gcc),$(CC))" TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-arm64-startup-bank.sh + +check-vir-native-arm64-stage0: $(OUT)/$(STAGE0) $(VIR_NATIVE_ARM64_FIXTURES) tests/vir-direct-arm64.sh tests/vir-direct-common.sh + TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-direct-arm64.sh 0 +check-vir-native-arm64-stage2: $(OUT)/$(STAGE2) $(VIR_NATIVE_ARM64_FIXTURES) tests/vir-direct-arm64.sh tests/vir-direct-common.sh + TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-direct-arm64.sh 2 +endif # Run the complete check -- driver and ABI suites at stages 0 and 2 -- on every # backend, as CI does. Driver cases that need 64-bit values are gated on the @@ -195,6 +292,25 @@ check-all-targets: $(MAKE) check ARCH="$$target_arch"; \ done +.PHONY: vir-baseline +vir-baseline: $(OUT)/$(STAGE0) tests/vir-baseline.sh tests/benchmark-time.sh \ + tests/vir-baseline-workloads.txt + VIR_BASELINE_RUNS=$${VIR_BASELINE_RUNS:-5} tests/vir-baseline.sh + +.PHONY: vir-opt-baseline +vir-opt-baseline: $(OUT)/$(STAGE0) tests/vir-opt-baseline.sh tests/benchmark-time.sh \ + tests/vir-opt-baseline-workloads.txt + tests/vir-opt-baseline.sh 0 + +.PHONY: vir-licm-profile +VIR_LICM_PROFILE_SAMPLES ?= 7 +VIR_LICM_PROFILE_REPEATS ?= 5 +vir-licm-profile: $(OUT)/vir-licm-profile + $(OUT)/vir-licm-profile $(VIR_LICM_PROFILE_SAMPLES) $(VIR_LICM_PROFILE_REPEATS) + +$(OUT)/vir-licm-profile: tests/vir-licm-profile.c src/vir.c src/vir.h + $(CC) $(CFLAGS) -Isrc tests/vir-licm-profile.c src/vir.c -o $@ + # One checker per target: they share nothing, so "make -j check-style" runs them # concurrently and finishes in the time the slowest one takes. check-style: check-newline check-comments check-format check-shell @@ -243,7 +359,7 @@ check-stage2: $(OUT)/$(STAGE2) tests/driver.sh $(VECHO) " TEST STAGE 2\n" tests/driver.sh 2 $(DYNLINK) $(BINDING) -check-sanitizer: $(OUT)/$(STAGE0)-sanitizer tests/driver.sh +check-sanitizer: $(OUT)/$(STAGE0)-sanitizer tests/driver.sh check-vir-sanitizer $(VECHO) " TEST STAGE 0 (with sanitizers)\n" $(Q)cp $(OUT)/$(STAGE0)-sanitizer $(OUT)/shecc tests/driver.sh 0 $(DYNLINK) $(BINDING) @@ -255,12 +371,81 @@ check-abi-stage0: $(OUT)/$(STAGE0) check-abi-stage2: $(OUT)/$(STAGE2) tests/$(ARCH)-abi.sh 2 $(DYNLINK) $(BINDING); +check-vir-core: tests/vir-core.c src/vir.c src/vir.h + $(CC) $(CFLAGS) -Isrc tests/vir-core.c src/vir.c -o $(OUT)/vir-core + $(OUT)/vir-core + +check-vir-frontend-stage0: $(OUT)/$(STAGE0) tests/vir-frontend.sh tests/vir-frontend-workloads.txt $(VIR_FRONTEND_FIXTURES) + TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-frontend.sh 0 + +check-vir-frontend-stage2: $(OUT)/$(STAGE2) tests/vir-frontend.sh tests/vir-frontend-workloads.txt $(VIR_FRONTEND_FIXTURES) + TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-frontend.sh 2 + +check-vir-frontend-policy-stage0: $(OUT)/$(STAGE0) tests/vir-frontend-policy.sh tests/vir-frontend-gvn.c tests/vir-frontend-licm.c tests/vir-frontend-nested-loop.c tests/vir-frontend-multi-latch.c + TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-frontend-policy.sh 0 + +check-vir-frontend-policy-stage2: $(OUT)/$(STAGE2) tests/vir-frontend-policy.sh tests/vir-frontend-gvn.c tests/vir-frontend-licm.c tests/vir-frontend-nested-loop.c tests/vir-frontend-multi-latch.c + TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-frontend-policy.sh 2 + +check-vir-fuzz: tests/vir-fuzz.c src/vir.c src/vir.h + $(CC) $(CFLAGS) -Isrc tests/vir-fuzz.c src/vir.c -o $(OUT)/vir-fuzz + $(OUT)/vir-fuzz + +check-vir-sanitizer: tests/vir-core.c tests/vir-fuzz.c src/vir.c src/vir.h + $(CC) $(CFLAGS) $(SAN_CFLAGS) -Isrc tests/vir-core.c src/vir.c -o $(OUT)/vir-core-sanitize + ASAN_OPTIONS=detect_leaks=1 $(OUT)/vir-core-sanitize + $(CC) $(CFLAGS) $(SAN_CFLAGS) -Isrc tests/vir-fuzz.c src/vir.c -o $(OUT)/vir-fuzz-sanitize + ASAN_OPTIONS=detect_leaks=1 $(OUT)/vir-fuzz-sanitize + +.PHONY: check-vir +check-vir: check-vir-core check-vir-frontend-stage0 \ + check-vir-frontend-stage2 check-vir-fuzz check-vir-stage0 check-vir-stage2 \ + check-vir-frontend-policy-stage0 \ + check-vir-frontend-policy-stage2 \ + check-vir-sanitizer + +check-vir-stage0: $(OUT)/$(STAGE0) tests/vir-stage0.c src/vir.c src/vir.h + $(OUT)/$(STAGE0) -o $(OUT)/vir-stage0 tests/vir-stage0.c + $(TARGET_EXEC) $(OUT)/vir-stage0 + +check-vir-stage2: $(OUT)/$(STAGE2) tests/vir-stage0.c src/vir.c src/vir.h + $(TARGET_EXEC) $(OUT)/$(STAGE2) -o $(OUT)/vir-stage2 tests/vir-stage0.c + $(TARGET_EXEC) $(OUT)/vir-stage2 + +.PHONY: check-vir-native-regressions-stage0 check-vir-native-regressions-stage2 +check: check-vir-native-regressions-stage0 check-vir-native-regressions-stage2 check-vir-lower +check-vir-native-regressions-stage0: $(OUT)/$(STAGE0) + TARGET_EXEC="$(TARGET_EXEC)" tests/vir-native-regressions.sh $(OUT)/$(STAGE0) +check-vir-native-regressions-stage2: $(OUT)/$(STAGE2) + TARGET_EXEC="$(TARGET_EXEC)" tests/vir-native-regressions.sh $(TARGET_EXEC) $(OUT)/$(STAGE2) + +.PHONY: check-vir-lower +check-vir-lower: $(OUT)/$(STAGE0) tests/vir-lower.c tests/vir-lower-smoke.c tests/vir-lower.sh + CC="$(CC)" TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-lower.sh + +check-vir-baseline: $(OUT)/$(STAGE0) tests/vir-baseline.sh tests/benchmark-time.sh \ + tests/vir-baseline-workloads.txt + VIR_BASELINE_RUNS=1 VIR_BASELINE_NO_TIMING=1 tests/vir-baseline.sh + +# Each workload is compiled twice at both levels by each stage, and the two +# compilers must agree on the output and on the VIR dump. The stats lines are +# left out because they report host-dependent arena sizes. +check-vir-opt-baseline: $(OUT)/$(STAGE0) $(OUT)/$(STAGE2) tests/vir-opt-baseline.sh \ + tests/benchmark-time.sh tests/vir-opt-baseline-workloads.txt + @set -e; stage0=$$(mktemp); stage2=$$(mktemp); trap 'rm -f "$$stage0" "$$stage2"' EXIT; \ + VIR_OPT_BASELINE_NO_TIMING=1 VIR_OPT_BASELINE_RUNS=2 tests/vir-opt-baseline.sh 0 > "$$stage0"; \ + TARGET_EXEC="$(TARGET_EXEC)" VIR_OPT_BASELINE_NO_TIMING=1 VIR_OPT_BASELINE_RUNS=2 tests/vir-opt-baseline.sh 2 > "$$stage2"; \ + sed -i 's/ stage=[02] / /' "$$stage0" "$$stage2"; \ + diff "$$stage0" "$$stage2" + # Both prerequisites are order-only, and both exist because "make -j" would # otherwise let a compile start beside the thing it reads. Selecting a target # replaces src/codegen.c with "ln -sf", which unlinks before it relinks, so a # compile racing "config" can find nothing there. And src/main.c includes # out/libc.inc, which is generated: listing it only on the stage0 link left # the two free to run in either order on a tree that has never been built. +$(OBJS) $(SAN_OBJS): $(OUT)/libc.inc + $(OUT)/%.o: %.c | config $(OUT)/libc.inc $(VECHO) " CC\t$@\n" $(Q)$(CC) -o $@ $(CFLAGS) -c -MMD -MF $@.d $< @@ -331,3 +516,9 @@ distclean: clean -$(RM) DOM.dot CFG.dot -include $(deps) + +# Manual native hardware measurement; excluded from correctness checks. +.PHONY: benchmark-native-throughput +benchmark-native-throughput: $(OUT)/$(STAGE0) + $(Q)python3 tests/native-throughput.py --shecc $(OUT)/$(STAGE0) \ + --output $(OUT)/native-throughput $(NATIVE_BENCH_FLAGS) diff --git a/README.md b/README.md index 8f7bc617..40be0cd5 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,9 @@ Despite its simplistic nature, it is capable of performing basic optimization st * Provide a minimal C standard library for basic I/O on GNU/Linux. * The cross-compiler is written in ANSI C, making it compatible with most platforms. * Include a self-contained C front-end with an integrated machine code generator; no external assembler or linker needed. -* Utilize a two-pass compilation process: the first pass checks syntax and breaks down complex statements into basic operations, - while the second pass translates these operations into target machine code. -* Develop a register allocation system that is compatible with RISC-style architectures. -* Implement an architecture-independent, [static single assignment](https://en.wikipedia.org/wiki/Static_single-assignment_form) (SSA)-based middle-end for enhanced optimizations. +* Use [VIR](docs/vir.md), a value-centric SSA representation, for target-independent optimization. +* Share ABI lowering, register allocation, and spill handling across all four targets. +* Lower VIR to allocated machine IR before target-specific instruction emission. * Support dynamic linking to allow generated executables to run with glibc. * Emit both ELF32 (Arm, RISC-V) and ELF64 (AArch64, x86-64) images; the ELF class follows the target pointer width. @@ -186,15 +185,19 @@ $ make check-sanitizer File `out/shecc` is the first stage compiler. Its usage: ```shell -$ shecc [-o output] [+m] [--dot] [--no-libc] [--dump-ir] [--dynlink] [-z ] [-E] +$ shecc [-o output] [+m] [--dot] [--no-libc] [--dump-ir] [--dump-vir] [--stats] [--no-opt] [--dynlink] [-z ] [-E] ``` Compiler options: - `-o` : Specify output file name (default: `out.elf`) - `+m` : Use hardware multiplication/division instructions (default: disabled) -- `--dot` : Write the SSA control-flow graph in Graphviz DOT format and stop +- `--dot` : Write the VIR control-flow graph in Graphviz DOT format and stop - `--no-libc` : Exclude embedded C library (default: embedded) - `--dump-ir` : Dump intermediate representation (IR) +- `--stats` : Print deterministic compiler and VIR counters +- `--no-opt` : Skip the optional VIR optimizations (default: enabled) +- VIR inspection and optimization options are + documented in the [VIR reference](docs/vir.md). - `--dynlink` : Use dynamic linking (default: disabled) - `-z`: Use lazy binding or immediate binding (default: lazy) - This option is valid only when `--dynlink` is also enabled. Otherwise, the compiler @@ -275,186 +278,45 @@ For resetting architecture configurations, use the command `make distclean`. ## Intermediate Representation -The middle end is SSA-based, and two options expose it. `--dot` draws the -control-flow graph as it stands after SSA construction and pruning; `--dump-ir` -prints the instructions as text, once before optimization and once after -register allocation. Both use `tests/fib.c` below as the running example, a -recursive Fibonacci function and its caller: - -```c -#include - -int fib(int n) -{ - if (n == 0) - return 0; - else if (n == 1) - return 1; - return fib(n - 1) + fib(n - 2); -} - -int main() -{ - printf("F(10) = %d\n", fib(10)); - return 0; -} -``` +[VIR](docs/vir.md) (Value Intermediate Representation) is shecc's native +value-centric SSA representation. The reference describes its invariants, +optimization schedule, shared machine lowering, and validation commands. ### Control-flow graph -`--dot` writes Graphviz DOT and stops; no executable is generated. Without -`-o`, the name is the input path with its final suffix replaced, or `.dot` -appended when it has none, so `tests/fib.c` would write `tests/fib.dot` into the -source tree. +`--dot` writes the optimized VIR control-flow graph as Graphviz DOT and stops +before executable generation. Without `-o`, the output name replaces the input +path's final suffix with `.dot`, or appends `.dot` when no suffix exists. ```shell $ out/shecc --dot -o fib.dot tests/fib.c $ dot -Tsvg fib.dot -o fib.svg ``` -Graphviz renders the result but is not needed to produce it, and nothing in -`make check` depends on it. - -The output holds one cluster per function definition, one nested cluster per -basic block, -and one node per IR instruction; a block holding no instruction gets a single -placeholder node labeled `pseudo`. Edges inside a block are instruction order, -and the edges leaving a block's last node are control flow. The graph is emitted -before phi values are unwound into edge copies, so the phi nodes are still in -it, and functions the input cannot reach are pruned first. That pruning is what -keeps the picture readable: with the embedded C library prepended, `fib.c` gives -48 function bodies, of which 22 are reachable. The 20 that are neither `fib` nor -`main` are what `printf` drags in transitively, down to `malloc` and `abort`. - -The figure below is the `fib` and `main` clusters of that output, with the -run-specific pointer values dropped from the labels. Those two are the last -clusters emitted, so keeping everything from `fib` to the end of the file is -exactly the input's own code; that is what makes the crude recipe below work, -not a general way to slice one function out: - -```shell -$ { echo 'strict digraph CFG {'; echo 'node [shape=box]' - awk '/\(fib\)"$/ { keep = 1; print prev } keep; { prev = $0 }' fib.dot; } | - sed -E 's/"(BasicBlock )?0x[0-9a-f]+ \((.+)\)"/"\2"/' > fib-fn.dot -$ dot -Tpng -Granksep=0.3 -Nfontname=Helvetica fib-fn.dot -o docs/fib-cfg.png -``` - -![SSA control-flow graph of tests/fib.c](docs/fib-cfg.png) - -Reading `fib` from the top: - -* `.label.1986` and `.label.1987` carry no instructions. Every function gets a - dedicated entry block and a single exit block; the entry block is empty only - when the body opens with a control-flow construct, as `fib` does with its - `if`, while `main` starts straight into instructions. All three `RETURN`s edge - into the exit, so the graph stays single-entry and single-exit no matter how - many `return` statements the C code spells out. -* `.label.1988` is the `n == 0` test: materialize the constant, compare, then - `BRANCH`. The true edge leaves the bottom-left corner of a `BRANCH` node and - the false edge the bottom-right; where the two targets land is up to Graphviz, - so the edges cross whenever it places them the other way around. -* `.label.1991` is `return 0`. `.label.1992` is the else arm, an empty block - that falls into `.label.1995`, the `n == 1` test. Those empty blocks are - branch targets the parser reserved before it knew what the arm would contain. -* `.label.1998` is `return 1`, and `.label.1999` falls into `.label.2002`, the - recursive tail. -* Each call in `.label.2002` is a `PUSH` per argument, a `CALL`, and a - `RETURN VALUE` naming the result. Argument passing stays explicit in the IR, - so the register allocator, not the front end, decides which registers carry - them. - -Subscripts in the figure are SSA versions: `n₀` is the incoming parameter, -`.t1247₀` the first definition of the temporary `.t1247`. `fib` needs no phi, -because control flow never merges after a write here; every value is defined -once on the path that reaches its use. A phi appears where it does merge. In the -same graph, `strlen` carries `i₂ := PHI(i₁, i₃)` at its loop header, one argument -per incoming edge: `i₁` from the block ahead of the loop, `i₃` from the back edge. - -Temporary and label numbers shift whenever `lib/c.c` changes. Read them as -identity within one dump, not as stable names. +Each node represents a basic block, labeled with its function name and block +number. Edges represent control flow between blocks. Only reachable functions +are included. Graphviz is needed to render the file, but not to produce it. ### Instruction dump ```shell $ out/shecc --dump-ir -o fib tests/fib.c +$ out/shecc --dump-vir -o fib tests/fib.c 2> fib.vir ``` -The first dump, delimited by `====` and -`====`, is the IR before optimization, walked in -dominator-tree order. Three things have already happened to it: small callees -are inlined into their callers, the phis are unwound into copies, and the -printer then elides those copies along with the SSA subscripts, so `--dot` is -the only view that shows either. Pruning has not run yet, so every function with -a body is listed, libc included. Line by line against the C source: - -``` -C source IR Notes ----------------------+---------------------------------------+--------------------------------- -int fib(int n) def int @fib(int %n) { %n parameter, %.tN temporary -{ - if (n == 0) const %.t1247, 0 - %.t1248 = eq %n, %.t1247 - br %.t1248, .label.1991, .label.1992 true target first, false second - return 0; .label.1991: - const %.t1249, 0 - ret %.t1249 - else if (n == 1) .label.1995: reached via empty .label.1992 - const %.t1250, 1 - %.t1251 = eq %n, %.t1250 - br %.t1251, .label.1998, .label.1999 - return 1; .label.1998: - const %.t1252, 1 - ret %.t1252 - return fib(n - 1) .label.2002: - const %.t1253, 1 - %.t1254 = sub %n, %.t1253 - push %.t1254 one push per argument - call @fib - retval %.t1255 names the callee's result - + fib(n - 2); const %.t1256, 2 - %.t1257 = sub %n, %.t1256 - push %.t1257 - call @fib - retval %.t1258 - %.t1259 = add %.t1255, %.t1258 - ret %.t1259 -} } -``` - -A block earns a label line only by holding instructions, and the first one that -qualifies is folded into the function header. That is why neither `.label.1992`, -an empty block the graph shows, nor `.label.1988`, which holds the first three -instructions, appears as a label here. The graph is the better view of control -flow; this dump is the better view of what each block computes. - -The second dump follows register allocation, the peephole pass, the -architecture-specific lowering, and the flattening of the CFG into a linear -list. It is the last IR before machine code: - -``` -fib: - store %x0, 16(sp) - li %x1, $0 - %x2 = eq %x0, %x1 - br %x2 - ... - li %x1, $2 - load %x2, 16(sp) - %x2 = sub %x2, %x1 - store %x0, 24(sp) - %x0 = %x2 - call @fib -``` - -The listing above is the default Arm configuration; the register numbers follow -whatever target `make config` selected. `%xN` are post-allocation register -slots rather than SSA values, and how many exist and which hardware register -each maps to is the backend's business. `16(sp)` is a stack slot: `n` is spilled -on entry because it stays live across both calls, and the first call's result is -spilled in turn while the second one runs. `br` prints only its condition, but -both targets are still on the instruction; the backend encodes the true one as an -offset and lets the false one fall through when it is the next block in the -list. +`--dump-ir` prints optimized VIR to standard output; `--dump-vir` prints it to +standard error. Each function starts with `function NAME`, followed by blocks, +typed SSA values, block parameters, and instructions. Calls carry their +arguments and signatures directly. Memory operations carry their widths, and +volatile operations remain explicit. The dump precedes shared register +allocation and machine lowering. + +Optimization is on by default; `--no-opt` turns the optional passes off. That +is for debugging and for isolating an optimizer fault, not for speed: the +unoptimized graph leaves more values to lower, so compilation is no faster. +Construction and final graph verification run either way. Value and block +numbers identify entities within a dump and can change as source or +optimization settings change. ## C99 Compliance diff --git a/docs/vir.md b/docs/vir.md new file mode 100644 index 00000000..a5cb1887 --- /dev/null +++ b/docs/vir.md @@ -0,0 +1,315 @@ +# VIR — Value Intermediate Representation + +VIR (Value Intermediate Representation) is shecc's target-independent, typed +intermediate representation. It uses static single assignment (SSA): each +scalar value has one definition, even when a source variable is assigned more +than once. Its organizing rule is: + +> CFG for control, graphs for values, order for effects. + +## Pipeline + +Every function and global initializer is constructed as VIR. The compiler +uses one pipeline for Armv7, AArch64, RV32, and x86-64: + +```mermaid +flowchart TD + C["C frontend: functions and global initializers"] + V["Typed VIR: control flow, values, ordered effects"] + O["Shared VIR optimizations"] + L["Shared lowering: ABI, register allocation, edge copies, spills"] + P["Allocated VIR machine operations"] + T["Machine peepholes and target lowering"] + E["Armv7 / AArch64 / RV32 / x86-64 emitter"] + ELF["ELF output"] + C --> V --> O --> L --> P --> T --> E --> ELF +``` + +The compiler optimizes and verifies each function's graph as soon as the +function is parsed, then copies the live graph into a compact arena; the global +initializer graph follows at the end of parsing. At O0, the compiler skips +optional transformations but still verifies. + +VIR owns values, control flow, and ordered effects. The shared lowerer assigns +physical registers, resolves edge copies, handles ABI boundaries, and allocates +spill storage. Its output is VIR's machine layer: `vir_mop_t` machine +operations, in machine blocks, that all four emitters consume. Function graphs +and analysis tables are released after lowering; the machine operations carry +every operand emission needs. + +Reachability follows calls and function addresses from `main` and global +initialization. Used functions must have definitions unless dynamic linking +can resolve them. Unsupported source forms and invalid graphs produce +compiler diagnostics. + +## Design: control, values, and effects + +A function graph (`vir_function_t`) combines three related structures: + +| Structure | Purpose | Main representation | +| --- | --- | --- | +| Control-flow graph (CFG) | Select which blocks execute and carry values between them. | `vir_block_t`, `vir_edge_t`, block parameters, and terminators | +| Value graph | Connect typed definitions to their operands and uses. | `vir_value_t`, operand links, and reverse `vir_use_t` links | +| Ordered effects | Preserve the order of memory accesses, calls, and volatile operations within each block. | `vir_effect_t`, with links to addresses, arguments, and results | + +These structures share values. A load or non-void call has both an effect entry +and a result value; a store consumes an address and a value but defines no +result. Pure computations have operand dependencies and a defining block, but +no entry in the effect list. A block ends with a jump, conditional branch, or +return. Effect order between blocks follows the executed CFG path. + +### Example: a value merged across branches + +Consider a function that selects one of two inputs, stores it, and returns its +successor: + +```c +int select_and_store(int condition, int a, int b, int *out) +{ + int selected; + if (condition) + selected = a; + else + selected = b; + *out = selected; + return selected + 1; +} +``` + +The following schematic shows its control flow before CFG simplification. +Names are explanatory labels, rather than literal VIR dump syntax. + +```mermaid +flowchart TD + ENTRY["entry: condition, a, b, out"] + THEN["then"] + ELSE["else"] + JOIN["join(x: i32): store(out, x); return x + 1"] + ENTRY -->|"condition != 0"| THEN + ENTRY -->|"condition == 0"| ELSE + THEN -->|"argument: a"| JOIN + ELSE -->|"argument: b"| JOIN +``` + +`x` is a block parameter: the incoming edge supplies either `a` or `b`. +The parameter is defined once at the join, and each edge argument must match +its type and position. This expresses the same merge as an SSA phi instruction. +During lowering, each edge's arguments become a parallel copy into the +locations assigned to the destination parameters. Copies whose source and +destination already share a location require no machine instruction. + +### Example: value dependencies and effect order + +Within a block, the value graph and effect list describe different constraints. +For `*out = *in + 1; notify();`, the load defines a value used by the addition, +and the store consumes the addition's result. The call follows the store in +the effect list: + +```mermaid +flowchart LR + IN["in: ptr"] + OUT["out: ptr"] + LOAD["load effect / result v: i32"] + ADD["sum = v + 1"] + STORE["store effect"] + CALL["call effect: notify()"] + IN -->|address| LOAD + LOAD -->|operand| ADD + ADD -->|stored value| STORE + OUT -->|address| STORE + LOAD -.->|effect order| STORE + STORE -.->|effect order| CALL +``` + +Solid arrows show value dependencies; dashed arrows show effect order. +The addition belongs to the value graph, while the load, store, and call also +belong to the block's effect list. This lets pure-value optimizations follow +operand links while memory transformations respect explicit effect boundaries. + +## Rationale + +VIR is SSA for scalar values: each value is defined once, and the verifier +checks dominance and use-def consistency. Its design choices concern how +merges, effects, and ABI information are represented alongside those values. + +### SSA representation choices + +- **Block parameters instead of phi instructions.** A jump edge carries + arguments and the receiving block declares parameters (`edge->args`, + `vir_block_add_param()`), as in MLIR and Cranelift. Adding, removing, + redirecting (`vir_edge_redirect()`) or splitting (`vir_edge_split()`) an + edge moves its arguments with it, so there is no phi operand list to keep in + step with the predecessor list. +- **Edge arguments define transfers.** Shared lowering resolves each edge as one + parallel copy. It materializes a copy block only when an edge needs one, + emits nothing when the arguments already sit in their parameters' registers, + and moves all-register edges without a round trip through spill homes. +- **Ordered memory effects.** Loads, stores, volatile accesses and calls form + an ordered effect list in each block, with their widths and types. That is + enough for the optimizer to decide which reorderings and removals are legal + without Memory SSA or general alias analysis, which keeps the compiler small + enough to stay self-hosting. +- **Calls carry ABI signatures.** Each call holds its arguments, result type, + and signature, including the fixed-parameter boundary for variadic calls. + Application binary interface (ABI) handling, including paired registers on + 32-bit targets, variadic saves, and results, lives in the shared lowering + rather than being duplicated across the four backends. +- **A standalone, verified core.** Unit tests (`tests/vir-core.c`) and a + structural fuzzer (`tests/vir-fuzz.c`) exercise the core independently of + the frontend. Every graph that is dumped or lowered is verified after + optimization, at every level, and dumps are deterministic. + +### Benefits and tradeoffs + +- **Fewer memory transfers at block boundaries.** In the legacy pipeline, + unwound phis were stored + to a stack slot and read straight back, and the register allocator gave up + its register state at block boundaries, so values crossing an edge went + through memory. Later passes such as `collapse_slot_roundtrip()` only undid + part of that after the fact. Block parameters let lowering keep those values + in registers from the start. +- **One lowering for four targets.** Register allocation, edge copies, ABI + boundaries and spill storage are target-independent, so a fix or an + improvement there reaches Armv7, AArch64, RV32 and x86-64 at once. Each + backend keeps instruction selection, encoding and its target-specific + folding and branch layout. +- **Optimization on a verified representation.** Constant propagation, + redundancy elimination, loop optimization, and dead-code elimination work on + typed values with explicit control flow and effect order, + and the verifier rejects a pass that breaks an invariant instead of letting + it reach code generation. +- **Measured result.** On the native throughput fixtures, generated code moved + from 30-81% of GCC `-O1` speed to 80-102% on x86-64; + [native throughput measurements](../tests/native-throughput/README.md) + record the workloads and host conditions. These synthetic fixtures are not a + general performance guarantee. The cost is compile time: self-compilation is slower than under the legacy + pipeline, though still several times faster than GCC `-O1`. +- **Still small and self-hosting.** VIR is plain C with deterministic + construction and no global interning, Memory SSA or alias-analysis + framework, so shecc remains a compiler a reader can follow end to end. + +## Representation contract + +- Values and blocks have stable integer IDs within a graph. Construction, + traversal, dumps, and lowering must be deterministic and independent of + host addresses and hash-table iteration order. +- Pure computations retain their defining block, insertion position, and + typed operands. Construction may fold constants, remove identities, and + order commutative operands when their semantics permit it. Common unary + and binary operands stay inline; unusual arities use side storage. +- Blocks and edges carry control. SSA merges use block parameters and edge + arguments. Edge mutation must preserve argument indices, types, and reverse + uses, including duplicate successors and loop backedges. +- Loads, stores, calls, and volatile accesses are ordered effects. Pure-value + CSE and LICM do not reorder or speculate them. DCE can remove unused ordinary + loads and exact same-address, same-type ordinary stores overwritten before + an observing effect. Used loads, calls, volatile accesses, and intervening + stores remain barriers. CFG rewrites may remove unreachable effects. +- Integer and pointer widths, alignment, constant bits, casts, and signed or + unsigned operations follow the selected target. Pointer values use target + pointer width. +- Arithmetic and signed or unsigned relational comparisons use i32/i64; + equality accepts same-typed values. i8/i16 describe ABI, cast, and memory + boundaries. The frontend applies C integer promotions before arithmetic. + Pointer loads retain pointer type; ABI metadata retains integer signedness + and `_Bool` identity. +- Source variables, SSA values, physical storage, and analysis data have + separate identities. Dominance, loops, SCCP, GVN, liveness, and allocation + tables belong to their passes and are invalidated after relevant CFG changes. + +## Compiler options + +VIR is the normal compilation path. Optimization is on by default (O2 below); +`--no-opt` selects O0. O0 exists for debugging and as a baseline for isolating +optimizer faults. It does not compile faster: the unoptimized graph leaves more +values for machine lowering. + +| Option | Behavior | +| --- | --- | +| `--dump-ir` | Print optimized live-function VIR to standard output. | +| `--dump-vir` | Print optimized live-function VIR to standard error. If both dump options are present, standard output is used. | +| `--dot` | Write the live-function VIR CFG in Graphviz DOT format and stop before machine emission. | +| `--stats` | Print deterministic layout, phase, VIR, machine-operation, and arena counters to standard error. | +| `--no-opt` | Skip the optional VIR optimizations (O0). | + +Textual graphs start with `function NAME`, followed by block definitions. +Tests resolve private functions and globals through their actual call and +address operands; private symbol spelling is not a source-language interface. + +`--no-libc` excludes the embedded C library. Compiler builtins remain +available. `--dynlink`, `+m`, preprocessing, and include-path options retain +their existing target-specific behavior. + +## Optimization and verification + +Frontend completion always seals SSA bindings and removes unreachable blocks. +Every graph is verified after optimization, including at O0, whether or not +anything references it. + +The compiler invokes these passes in order. Sparse conditional constant +propagation (SCCP) combines constant propagation with reachability analysis. +Common subexpression elimination (CSE) and global value numbering (GVN) remove +redundant computations. Reassociation moves a constant off a block-local operand +onto one computed earlier, so it leaves a loop-carried chain. Loop-invariant +code motion (LICM) moves eligible pure computations out of loops, and dead-code +elimination (DCE) removes unused work. + +1. Remove unreachable blocks. +2. SCCP and CFG simplification. +3. Local CSE. +4. GVN. +5. Reassociation, LICM and pointer-induction strength reduction. +6. Local CSE and DCE, including ordinary overwritten-store elimination. +7. CFG simplification, then a final local CSE and DCE over the simplified graph. +8. Full graph verification before dumps and machine lowering. + +| Level | Scheduled transformations | +| --- | --- | +| O0 | Construction folds, unreachable-block removal, and mandatory verification; optional optimization passes return without transforming the graph. | +| O2 (default) | SCCP, CFG simplification, local CSE, GVN, reassociation, LICM, strength reduction, and DCE. | + +Pass-local tables are temporary. O0 does not allocate the optional CSE, GVN, +loop, or optimization worklists. Mandatory verification still checks dominance, +types, use-def links, edges, and ordered effects. + +## Validation + +| Command | Coverage | +| --- | --- | +| `make check-vir-core` | Builders, core invariants, and transformations. | +| `make check-vir-lower` | Shared allocation and lowering invariants, plus an executable smoke test for the configured target. | +| `make check-vir` | Core, frontend, optimization policy, self-hosted VIR, fuzzing, and sanitizer gates. | +| `make check` | Full configured-target stage 0/2 behavior, ABI, and VIR checks. | +| `make check-all-targets` | Bootstrap and regression sweep across all four targets, using each configured runner. | +| `make check-vir-native-x64-stage0 check-vir-native-x64-stage2` | Native x86-64 execution and graph quality checks. | +| `make check-vir-native-arm64-stage0 check-vir-native-arm64-stage2` | Native AArch64 execution and graph quality checks using the configured `TARGET_EXEC`. | +| `make check-vir-native-regressions-stage0 check-vir-native-regressions-stage2` | Native frontend, ABI, and runtime regression fixtures with both compiler stages. | +| `make check-vir-multi-input-stage0 check-vir-multi-input-stage2` | Translation-unit isolation, private symbol identity, and cross-input linking. | +| `make check-vir-corpus-stage0` | Execution equivalence of the default build against O0 across the C fixture corpus. | +| `bash tests/vir-qualifiers.sh out/shecc` | Pointer and volatile qualifier execution and graph checks at O0 and the default level. | + +Corpus compilation defaults to 300 seconds per build. Set +`VIR_CORPUS_COMPILE_TIMEOUT` for slower self-compiled runs under emulation; +program execution retains its 10-second limit. + +Changes must preserve execution, deterministic dumps and output, a clean +verifier, and stage-1/stage-2 bootstrap identity. The AArch64 native harness +adds its default dynamic QEMU sysroot only when the configured runner has not +already supplied `-L`. + +## Implementation map + +| Concern | Location | +| --- | --- | +| Values, CFG, builders, verification, and optimizations | `src/vir.h`, `src/vir.c` | +| Native frontend and sealed SSA construction | `src/vir-frontend.c`, `src/vir-frontend.h` | +| Shared ABI lowering, register allocation, and spills | `src/vir-lower.c`, `src/vir-lower.h` | +| Machine emission | `src/arm-codegen.c`, `src/arm64-codegen.c`, `src/riscv-codegen.c`, `src/x64-codegen.c` | +| Pass scheduling and graph compaction | `src/vir-frontend.c` | +| Reachability, dumps, and graph lifetime | `src/main.c` | +| Core and frontend fixtures | `tests/vir-core.c`, `tests/vir-frontend-*.c`, and their manifests | +| Native execution and graph quality harnesses | `tests/vir-direct-*.sh`, `tests/vir-qualifiers.sh` | +| Aggregate and bootstrap gates | `Makefile` targets beginning with `check-vir` | + +This page defines the current VIR contract and validation entry points. Keep +the project overview in [README.md](../README.md) linked to this reference. diff --git a/lib/c.c b/lib/c.c index ce6ac3fc..12119669 100644 --- a/lib/c.c +++ b/lib/c.c @@ -10,9 +10,8 @@ /* Staging buffer for the printf family that writes straight to a descriptor. * - * The longest single call in the tree is ssa.c's "insn_%p [label=%s]": a - * DUMP_INSN_LEN staging buffer plus 26 bytes around it, so 537. Every byte here - * is stack in every program shecc emits, so it stays close to that. + * Keep the buffer bounded: every byte here occupies stack space in programs + * compiled by shecc. */ #define FMT_BUF_LEN 576 @@ -792,27 +791,27 @@ int fputc(int c, FILE *stream) int fseek(FILE *stream, int offset, int whence) { - int result; - /* RV32 has only _llseek, which splits the offset and returns the result * through a pointer. Every other target takes (fd, offset, whence) * directly. lib/c.h rejects an architecture that is neither. */ #if defined(__riscv) - result = __syscall(__syscall_lseek, stream, 0, offset, NULL, whence); + long long position; + return __syscall(__syscall_lseek, stream, offset < 0 ? -1 : 0, offset, + &position, whence) < 0; #else - result = __syscall(__syscall_lseek, stream, offset, whence); + return __syscall(__syscall_lseek, stream, offset, whence) < 0; #endif - return result == -1; } int ftell(FILE *stream) { /* See fseek(): only RV32 needs the split-offset _llseek form. */ #if defined(__riscv) - int result; - __syscall(__syscall_lseek, stream, 0, 0, &result, SEEK_CUR); - return result; + long long result; + if (__syscall(__syscall_lseek, stream, 0, 0, &result, SEEK_CUR) < 0) + return -1; + return (int) result; #else return __syscall(__syscall_lseek, stream, 0, SEEK_CUR); #endif @@ -843,13 +842,35 @@ int __align_up(int size) return ALIGN_UP(size, PAGESIZE); } +/* The raw mmap2 system call reports failure as -errno, -4095 to -1, not the -1 + * of libc's wrapper. Success is always page aligned and those values never are, + * so the low bits alone tell them apart on every word size. + */ +#define __mmap_failed(addr) (((int) (addr) & (PAGESIZE - 1)) != 0) + chunk_t *__alloc_head; chunk_t *__alloc_tail; -chunk_t *__freelist_head; + +/* Freed chunks, binned by size and linked through next. Every chunk is a whole + * number of pages, so the chunks in one bin are all the same size and the first + * non-empty bin at or above a request holds its best fit. A single list kept + * every freed chunk, so a program that frees as it goes scanned all of them on + * every allocation. Chunks are never split, so a request takes nothing twice + * its size or more, and chunks too large for a bin go back to the kernel. + */ +#define __FREE_BIN_PAGES 128 +chunk_t *__free_bins[__FREE_BIN_PAGES]; + +int __free_bin(int size) +{ + return size / PAGESIZE; +} void *malloc(int size) { - if (size <= 0) + /* Rounding up to MIN_ALIGNMENT and then to whole pages must not wrap. */ + if (size <= 0 || + size > INT_MAX - PAGESIZE - MIN_ALIGNMENT - (int) sizeof(chunk_t)) return NULL; int flags = 34; /* MAP_PRIVATE (0x02) | MAP_ANONYMOUS (0x20) */ @@ -862,7 +883,7 @@ void *malloc(int size) chunk_t *tmp = (chunk_t *) __syscall(__syscall_mmap2, NULL, __align_up(sizeof(chunk_t)), prot, flags, -1, 0); - if (tmp == (void *) -1) + if (__mmap_failed(tmp)) return NULL; __alloc_head = tmp; __alloc_tail = tmp; @@ -871,55 +892,25 @@ void *malloc(int size) __alloc_head->size = 0; } - if (!__freelist_head) { - chunk_t *tmp = (chunk_t *) __syscall(__syscall_mmap2, NULL, - __align_up(sizeof(chunk_t)), prot, - flags, -1, 0); - if (tmp == (void *) -1) - return NULL; - __freelist_head = tmp; - __freelist_head->next = NULL; - __freelist_head->prev = NULL; - __freelist_head->size = -1; - } + int needed = __align_up(sizeof(chunk_t) + size); + int bin = __free_bin(needed); + int last = bin * 2 < __FREE_BIN_PAGES ? bin * 2 : __FREE_BIN_PAGES; + chunk_t *allocated = NULL; - /* Search for the best fit chunk in the free list */ - chunk_t *best_fit_chunk = NULL; - chunk_t *allocated; - int best_size = 0; - - if (!__freelist_head->next) { - allocated = NULL; - } else { - for (chunk_t *fh = __freelist_head; fh->next; fh = fh->next) { - int fh_size = CHUNK_GET_SIZE(fh->size); - if (fh_size >= size && (!best_fit_chunk || fh_size < best_size)) { - best_fit_chunk = fh; - best_size = fh_size; - } - } - - if (best_fit_chunk) { - /* Remove from freelist */ - if (best_fit_chunk->prev) { - best_fit_chunk->prev->next = best_fit_chunk->next; - } else { - __freelist_head = best_fit_chunk->next; - } - if (best_fit_chunk->next) { - best_fit_chunk->next->prev = best_fit_chunk->prev; - } + for (; bin < last; bin++) { + if (__free_bins[bin]) { + allocated = __free_bins[bin]; + __free_bins[bin] = allocated->next; + break; } - allocated = best_fit_chunk; } if (!allocated) { - allocated = (chunk_t *) __syscall(__syscall_mmap2, NULL, - __align_up(sizeof(chunk_t) + size), - prot, flags, -1, 0); - if (allocated == (void *) -1) + allocated = (chunk_t *) __syscall(__syscall_mmap2, NULL, needed, prot, + flags, -1, 0); + if (__mmap_failed(allocated)) return NULL; - allocated->size = __align_up(sizeof(chunk_t) + size); + allocated->size = needed; } /* Add to allocation list */ @@ -960,21 +951,20 @@ void __rfree(void *ptr, int size) int __free_all(void) { - if (!__freelist_head && !__alloc_head) - return 0; - - chunk_t *cur = __freelist_head; + chunk_t *cur; chunk_t *rel; int size; - /* release freelist */ - while (cur && cur->next) { - rel = cur; - cur = cur->next; - rel->next = NULL; - rel->prev = NULL; - size = CHUNK_GET_SIZE(rel->size); - __rfree(rel, size); + /* release the freed chunks */ + for (int bin = 0; bin < __FREE_BIN_PAGES; bin++) { + cur = __free_bins[bin]; + while (cur) { + rel = cur; + cur = cur->next; + size = CHUNK_GET_SIZE(rel->size); + __rfree(rel, size); + } + __free_bins[bin] = NULL; } if (__alloc_head && __alloc_head->next) { @@ -1015,16 +1005,23 @@ void free(void *ptr) if (cur->next) { chunk_t *next = cur->next; next->prev = cur->prev; - } else if (prev) { - prev->next = NULL; + } else { + if (prev) + prev->next = NULL; __alloc_tail = prev; } - /* Insert head in __freelist_head */ - cur->next = __freelist_head; - cur->prev = NULL; + int size = CHUNK_GET_SIZE(cur->size); + int bin = __free_bin(size); + if (bin >= __FREE_BIN_PAGES) { + __rfree(cur, size); + return; + } + + /* Most recently freed first, so equal-sized chunks are reused in that order + * as before. + */ + cur->next = __free_bins[bin]; chunk_set_freed(cur); - if (__freelist_head) - __freelist_head->prev = cur; - __freelist_head = cur; + __free_bins[bin] = cur; } diff --git a/mk/arm.mk b/mk/arm.mk index f1f33711..8ffa1b90 100644 --- a/mk/arm.mk +++ b/mk/arm.mk @@ -20,6 +20,7 @@ ARCH_DEFS = \ \#define RESERVED_GOT_NUM 3\n$\ \#define R_ARCH_JUMP_SLOT 0x16\n$\ \#define MAX_ARGS_IN_REG 4\n$\ + \#define CALLEE_SAVED_REGS 4 /* r4-r7: AAPCS, and __syscall restores them */\n$\ " # If the running machine has the "fastfetch" tool installed, the build diff --git a/mk/arm64.mk b/mk/arm64.mk index 9eab2908..69c2076b 100644 --- a/mk/arm64.mk +++ b/mk/arm64.mk @@ -17,8 +17,8 @@ ARCH_DEFS = \ * gap leaves both inside one 64K page, where the writable mapping replaces\n$\ * the executable one. */\n$\ \#define PAGESIZE 65536\n$\ - \#define REG_CNT 11\n$\ - \#define CALLEE_SAVED_REGS 3\n$\ + \#define REG_CNT 24\n$\ + \#define CALLEE_SAVED_REGS 9\n$\ \#define DYN_LINKER \"/lib/ld-linux-aarch64.so.1\"\n$\ \#define LIBC_SO \"libc.so.6\"\n$\ \#define LIBDL_SO \"libdl.so.2\"\n$\ diff --git a/mk/riscv.mk b/mk/riscv.mk index 53402b5b..4b3eaba7 100644 --- a/mk/riscv.mk +++ b/mk/riscv.mk @@ -18,6 +18,8 @@ ARCH_DEFS = \ \#define RESERVED_GOT_NUM 2\n$\ \#define R_ARCH_JUMP_SLOT 0x5\n$\ \#define MAX_ARGS_IN_REG 8\n$\ + \#define REG_CNT 17 /* a0-a7, then s3-s11 */\n$\ + \#define CALLEE_SAVED_REGS 9 /* s3-s11 */\n$\ " TOOLCHAIN_CANDIDATES = riscv32-unknown-linux-gnu- diff --git a/mk/x64.mk b/mk/x64.mk index 877c1ed8..9efff718 100644 --- a/mk/x64.mk +++ b/mk/x64.mk @@ -17,6 +17,5 @@ ARCH_DEFS = \ \#define RESERVED_GOT_NUM 3\n$\ \#define R_ARCH_JUMP_SLOT 7 /* R_X86_64_JUMP_SLOT */\n$\ \#define REG_CNT 11 /* rdi rsi rdx rcx r8 r9 rax rbx r14 r12 r13 */\n$\ - \#define HAVE_COND_MOVE 1 /* CMOVcc */\n$\ \#define CALLEE_SAVED_REGS 4 /* the file ends rbx r14 r12 r13 */\n$\ " diff --git a/src/arch-lower.c b/src/arch-lower.c deleted file mode 100644 index d8ee19be..00000000 --- a/src/arch-lower.c +++ /dev/null @@ -1,37 +0,0 @@ -/* - * shecc - Architecture-specific IR lowering stage - * - * Introduces a minimal arch-lowering boundary that applies target-specific - * tweaks to phase-2 IR (ph2_ir) before final code generation. This keeps - * backends simpler by moving decisions that depend on CFG shape or target - * quirks out of emit-time where possible. - */ - -#include "../config" -#include "defs.h" - -/* Mark detached conditional branches so codegen can decide between short/long - * forms without re-deriving CFG shape. - * - * Only the ARM backend reads 'is_branch_detached'; RISC-V and x86-64 ignore it, - * so the pass runs for ARM alone. - */ -void arch_lower(void) -{ -#if ELF_MACHINE == ELF_MACHINE_ARM32 - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (ph2_ir_t *insn = bb->ph2_ir_list.head; insn; - insn = insn->next) { - /* In SSA, we index 'else_bb' first, and then 'then_bb' */ - if (insn->op == OP_branch) - insn->is_branch_detached = (insn->else_bb != bb->rpo_next); - } - } - } -#endif -} diff --git a/src/arm-codegen.c b/src/arm-codegen.c index 359c4ab5..35e70653 100644 --- a/src/arm-codegen.c +++ b/src/arm-codegen.c @@ -17,267 +17,58 @@ * addresses its high word four bytes further up. The size estimator and the * emitter both ask this, so the two cannot disagree. */ -bool arm_offset_needs_movw(const ph2_ir_t *ph2_ir) +bool arm_offset_needs_movw(const vir_mop_t *mop) { - bool load = ph2_ir->op == OP_load || ph2_ir->op == OP_global_load; - int offset = load ? ph2_ir->src0 : ph2_ir->src1; - bool eight_bits = ph2_ir->size_bytes == 2 || - (load && ph2_ir->size_bytes == 1 && !ph2_ir->is_unsigned); + bool load = mop->op == OP_load || mop->op == OP_global_load; + int offset = load ? mop->src0 : mop->src1; + bool eight_bits = mop->size_bytes == 2 || + (load && mop->size_bytes == 1 && !mop->is_unsigned); - if ((load ? ph2_ir->dest_hi : ph2_ir->src0_hi) >= 0) + if ((load ? mop->dest_hi : mop->src0_hi) >= 0) return offset > 4091; return offset > (eight_bits ? 255 : 4095); } -void update_elf_offset(ph2_ir_t *ph2_ir) +void update_elf_offset(vir_mop_t *mop) { - func_t *func; - switch (ph2_ir->op) { - case OP_load_constant: - /* ARMv7 uses 12 bits to encode immediate value, but the higher 4 bits - * are for rotation. See A5.2.4 "Modified immediate constants in ARM - * instructions" in ARMv7-A manual. - */ - if (ph2_ir->src0 < 0) - elf_offset += 12; - else if (ph2_ir->src0 > 255) - elf_offset += 8; - else - elf_offset += 4; - if (ph2_ir->dest_hi >= 0) { - if (ph2_ir->src1 < 0) - elf_offset += 12; - else if (ph2_ir->src1 > 255) - elf_offset += 8; - else - elf_offset += 4; - } - return; - case OP_address_of: - case OP_global_address_of: - /* ARMv7 uses 12 bits to encode immediate value, but the higher 4 bits - * are for rotation. See A5.2.4 "Modified immediate constants in ARM - * instructions" in ARMv7-A manual. - */ - if (ph2_ir->src0 > 255) - elf_offset += 12; - else if (ph2_ir->src0 >= 0) - elf_offset += 4; - else - abort(); - return; - case OP_assign: - if (ph2_ir->dest != ph2_ir->src0) - elf_offset += 4; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->dest_hi != ph2_ir->src0_hi) - elf_offset += 4; - /* Exchanging the two registers of a pair takes a third move. */ - if (ph2_ir->dest_hi >= 0 && ph2_ir->dest == ph2_ir->src0_hi && - ph2_ir->dest_hi == ph2_ir->src0) - elf_offset += 4; - return; - case OP_load: - case OP_global_load: - /* LDR/LDRB use a 12-bit offset, but LDRSB and LDRH/LDRSH use eight - * bits. A larger signed-byte or halfword offset is materialized with - * MOVW/MOVT/ADD before the load, exactly as an offset beyond the - * general load range is. A wide pair also loads its high word four - * bytes further up. - */ - if (arm_offset_needs_movw(ph2_ir)) - if (ph2_ir->dest_hi >= 0) - elf_offset += 20; - else - elf_offset += 16; - else if (ph2_ir->src0 >= 0) - if (ph2_ir->dest_hi >= 0) - elf_offset += 8; - else - elf_offset += 4; - else - abort(); - return; - case OP_store: - case OP_global_store: - /* ARMv7 straight uses 12 bits to encode the offset of store instruction - * (no rotation), but STRH only eight. A wide pair also stores its high - * word four bytes further up. - */ - if (arm_offset_needs_movw(ph2_ir)) - if (ph2_ir->src0_hi >= 0) - elf_offset += 20; - else - elf_offset += 16; - else if (ph2_ir->src1 >= 0) - if (ph2_ir->src0_hi >= 0) - elf_offset += 8; - else - elf_offset += 4; - else - abort(); - return; - case OP_read: - elf_offset += ph2_ir->dest_hi >= 0 ? 8 : 4; - return; - case OP_write: - elf_offset += ph2_ir->src1_hi >= 0 ? 8 : 4; - return; - case OP_indirect: - /* blx, then under dynamic linking the r12 reload that OP_call adds. */ - elf_offset += dynlink ? 16 : 4; - return; - case OP_jump: - case OP_load_func: - case OP_lshift: - case OP_rshift: - elf_offset += ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 ? 48 : 4; - return; - case OP_mul: - elf_offset += - ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && ph2_ir->src1_hi >= 0 - ? 20 - : 4; - return; - case OP_negate: - elf_offset += 4; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) - elf_offset += 4; - return; - case OP_add: - case OP_sub: - elf_offset += 4; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - elf_offset += 4; - return; - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: - elf_offset += 4; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - elf_offset += 4; - return; - case OP_bit_not: - elf_offset += 4; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) - elf_offset += 4; - return; - case OP_call: - func = find_func(ph2_ir->func_name); - if (func->bbs) - elf_offset += 4; - else if (dynlink) { - /* When calling external functions in dynamic linking mode, the - * following instructions are required: - * - movw + movt: set r8 to 'elf_data_start' - * - ldr: load a word from the address 'elf_data_start' into r12. - * (restore the global stack pointer.) - * - * Therefore, the total offset is 16 bytes (4 instructions). - */ - elf_offset += 16; - } else { - printf("The '%s' function is not implemented\n", ph2_ir->func_name); - fflush(stdout); /* see fatal() */ - abort(); - } - return; - case OP_div: - case OP_mod: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) { - /* The stack-backed restoring loop below has a fixed 64-round body. - * It keeps all pair state out of operand registers: an SSA result - * may coalesce with either dying input pair. - */ - if (ph2_ir->src0_is_unsigned || ph2_ir->src1_is_unsigned) - elf_offset += 284; - else - elf_offset += ph2_ir->op == OP_div ? 424 : 416; - return; - } - if (hard_mul_div) { - if (ph2_ir->op == OP_div) - elf_offset += 4; - else - elf_offset += 12; - return; + elf_offset += mop_count_words(mop) * 4; +} + +static mop_frame_layout_t arm_frame_layout(func_t *func, vir_mop_t *define) +{ + /* Eight-byte alignment follows the nine saved words and one padding word. + */ + return (mop_frame_layout_t) {ALIGN_UP(func->stack_size, MIN_ALIGNMENT) + 40, + mop_count_words(define) * 4, 0, false}; +} + +/* Mark the edges that fall into the block laid out next, before offsets are + * planned: a jump there emits nothing, and a branch tests for one edge alone. A + * branch keeps 1 when its else block follows, 2 when its then block does, and 0 + * when neither does, so both edges need a branch. + */ +static void arm_mark_fallthrough(void) +{ + FOR_EACH_FUNCTION_BODY(func) + { + for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { + for (vir_mop_t *mop = bb->mops.head; mop; mop = mop->next) { + if (mop->op == OP_branch) + mop->src2 = mop->else_bb == bb->rpo_next ? 1 + : mop->then_bb == bb->rpo_next ? 2 + : 0; + else if (mop->op == OP_jump && !mop->next) + mop->src2 = mop->next_bb == bb->rpo_next; + } } - /* div/mod emulation's offset */ - elf_offset += 124; - return; - case OP_load_data_address: - case OP_load_rodata_address: - elf_offset += 8; - return; - case OP_address_of_func: - elf_offset += 12; - return; - case OP_log_not: - elf_offset += ph2_ir->src0_hi >= 0 ? 16 : 12; - return; - case OP_gt: - case OP_lt: - case OP_geq: - case OP_leq: - elf_offset += ph2_ir->src0_hi >= 0 && ph2_ir->src1_hi >= 0 ? 16 : 12; - return; - case OP_eq: - case OP_neq: - elf_offset += ph2_ir->src0_hi >= 0 && ph2_ir->src1_hi >= 0 ? 16 : 12; - return; - case OP_branch: - if (ph2_ir->is_branch_detached) - elf_offset += 12; - else - elf_offset += 8; - if (ph2_ir->src0_hi >= 0) - elf_offset += 4; - return; - case OP_return: - elf_offset += 24; - if (ph2_ir->src0_hi >= 0) - elf_offset += 4; - return; - case OP_trunc: - /* Unsigned byte/half truncation uses a logical shift pair; signed - * narrowing uses the single-instruction SXTB/SXTH forms. - */ - if (ph2_ir->is_unsigned && (ph2_ir->src1 == 1 || ph2_ir->src1 == 2)) - elf_offset += 8; - else - elf_offset += 4; - return; - case OP_sign_ext: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi < 0) { - int source_size = (ph2_ir->src1 >> 16) & 0xFFFF; - elf_offset += - (ph2_ir->src0_is_unsigned || ph2_ir->src0_is_pointer) && - (source_size == 1 || source_size == 2) - ? 12 - : 8; - } else if (ph2_ir->src0_is_unsigned) - elf_offset += 8; - else - elf_offset += 4; - return; - case OP_cast: - elf_offset += - ph2_ir->dest_hi >= 0 && - (ph2_ir->src0_hi < 0 || ph2_ir->dest_hi != ph2_ir->src0_hi) - ? 8 - : 4; - return; - default: - fatal("Unknown opcode"); } } void cfg_flatten(void) { func_t *func; + mop_flat_prepare(false); + arm_mark_fallthrough(); if (dynlink) elf_offset = 132; /* __libc_start_main call, main_wrapper, and the @@ -291,10 +82,7 @@ void cfg_flatten(void) GLOBAL_FUNC->bbs->elf_offset = elf_offset; - for (ph2_ir_t *ph2_ir = GLOBAL_FUNC->bbs->ph2_ir_list.head; ph2_ir; - ph2_ir = ph2_ir->next) { - update_elf_offset(ph2_ir); - } + mop_visit_globals(update_elf_offset); /* prepare 'argc' and 'argv', then proceed to 'main' function */ if (dynlink) @@ -302,83 +90,40 @@ void cfg_flatten(void) else elf_offset += 32; /* 6 insns for main call + 2 for exit */ - for (func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - /* reserve stack */ - ph2_ir_t *flatten_ir = add_ph2_ir(OP_define); - flatten_ir->src0 = func->stack_size; - flatten_ir->func_name = intern_string(func->return_def.var_name); - - /* The actual offset of the top of the local stack is the sum of: - * - 36 bytes (pushing registers r4-r11 and lr onto the stack) - * - 4 bytes - * (to ensure 8-byte alignment after pushing the 9 registers) - * - ALIGN_UP(func->stack_size, 8) - * - * Note that func->stack_size does not include the 36 + 4 bytes, so an - * additional 40 bytes should be added to ALIGN_UP(func->stack_size, 8). - */ - int stack_top_ofs = ALIGN_UP(func->stack_size, MIN_ALIGNMENT) + 40; - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - bb->elf_offset = elf_offset; - - if (bb == func->bbs) { - /* retrieve the global stack pointer and save ra, sp */ - elf_offset += 28; - } - - for (ph2_ir_t *insn = bb->ph2_ir_list.head; insn; - insn = insn->next) { - /* For the instructions whose ofs_based_on_stack_top is set, - * recalculate the operand's offset by adding stack_top_ofs. - */ - if (insn->ofs_based_on_stack_top) { - switch (insn->op) { - case OP_load: - case OP_address_of: - insn->src0 = insn->src0 + stack_top_ofs; - break; - case OP_store: - insn->src1 = insn->src1 + stack_top_ofs; - break; - default: - /* Ignore opcodes with the ofs_based_on_stack_top flag - * set since only the three opcodes above needs to - * access a variable's address. - */ - break; - } - } - flatten_ir = add_existed_ph2_ir(insn); - - if (insn->op == OP_return) { - /* restore sp */ - flatten_ir->src1 = bb->belong_to->stack_size; - } - - /* Branch detachment is determined in the arch-lowering stage */ - - update_elf_offset(flatten_ir); - } - } - } + mop_flatten_functions(arm_frame_layout, update_elf_offset); } void emit(int code) { - elf_write_int(elf_code, code); + if (mop_count_only) + ++mop_counted_words; + else + elf_write_int(elf_code, code); } -void emit_ph2_ir(ph2_ir_t *ph2_ir) +static int arm_bitwise_insn(opcode_t op, + arm_reg dest, + arm_reg left, + arm_reg right) { - func_t *func; - const int rd = ph2_ir->dest; - const int rn = ph2_ir->src0; - int rm = ph2_ir->src1; /* Not const because OP_trunc modifies it */ + switch (op) { + case OP_bit_and: + return __and_r(__AL, dest, left, right); + case OP_bit_or: + return __or_r(__AL, dest, left, right); + case OP_bit_xor: + return __eor_r(__AL, dest, left, right); + default: + fatal("unsupported ARM bitwise operation"); + } + return 0; +} + +void emit_mop(vir_mop_t *mop) +{ + const int rd = mop->dest; + const int rn = mop->src0; + int rm = mop->src1; /* Not const because OP_trunc modifies it */ int ofs; int store_offset; bool is_external_call = false; @@ -390,9 +135,9 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) */ arm_reg interm; - switch (ph2_ir->op) { + switch (mop->op) { case OP_define: - fatal_function_context = ph2_ir->func_name; + fatal_function_context = mop->func_name; /* We should handle the function entry point carefully due to the * following constraints: @@ -409,91 +154,89 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) * 2. retrieve the global stack pointer from the 4-byte global object * located at 'elf_data_start' to ensure correct access to the global * stack. - * 3. set ofs to align(ph2_ir->src0, 8) + 4, and prepare a local + * 3. set ofs to align(mop->src0, 8) + 4, and prepare a local * stack for the callee by subtracting ofs from sp. */ emit(__stmdb(__AL, 1, __sp, 0x4FF0)); emit(__movw(__AL, __r8, elf_data_start)); emit(__movt(__AL, __r8, elf_data_start)); emit(__lw(__AL, __r12, __r8, 0)); - ofs = ALIGN_UP(ph2_ir->src0, MIN_ALIGNMENT) + 4; + ofs = ALIGN_UP(mop->src0, MIN_ALIGNMENT) + 4; emit(__movw(__AL, __r8, ofs)); emit(__movt(__AL, __r8, ofs)); emit(__sub_r(__AL, __sp, __sp, __r8)); return; case OP_load_constant: - if (ph2_ir->src0 < 0) { - emit(__movw(__AL, __r8, -ph2_ir->src0)); - emit(__movt(__AL, __r8, -ph2_ir->src0)); + if (mop->src0 < 0) { + emit(__movw(__AL, __r8, -mop->src0)); + emit(__movt(__AL, __r8, -mop->src0)); emit(__rsb_i(__AL, rd, 0, __r8)); - } else if (ph2_ir->src0 > 255) { - emit(__movw(__AL, rd, ph2_ir->src0)); - emit(__movt(__AL, rd, ph2_ir->src0)); + } else if (mop->src0 > 255) { + emit(__movw(__AL, rd, mop->src0)); + emit(__movt(__AL, rd, mop->src0)); } else - emit(__mov_i(__AL, rd, ph2_ir->src0)); - if (ph2_ir->dest_hi >= 0) { - if (ph2_ir->src1 < 0) { - emit(__movw(__AL, __r8, -ph2_ir->src1)); - emit(__movt(__AL, __r8, -ph2_ir->src1)); - emit(__rsb_i(__AL, ph2_ir->dest_hi, 0, __r8)); - } else if (ph2_ir->src1 > 255) { - emit(__movw(__AL, ph2_ir->dest_hi, ph2_ir->src1)); - emit(__movt(__AL, ph2_ir->dest_hi, ph2_ir->src1)); + emit(__mov_i(__AL, rd, mop->src0)); + if (mop->dest_hi >= 0) { + if (mop->src1 < 0) { + emit(__movw(__AL, __r8, -mop->src1)); + emit(__movt(__AL, __r8, -mop->src1)); + emit(__rsb_i(__AL, mop->dest_hi, 0, __r8)); + } else if (mop->src1 > 255) { + emit(__movw(__AL, mop->dest_hi, mop->src1)); + emit(__movt(__AL, mop->dest_hi, mop->src1)); } else - emit(__mov_i(__AL, ph2_ir->dest_hi, ph2_ir->src1)); + emit(__mov_i(__AL, mop->dest_hi, mop->src1)); } return; case OP_address_of: case OP_global_address_of: - interm = ph2_ir->op == OP_address_of ? __sp : __r12; - if (ph2_ir->src0 > 255) { - emit(__movw(__AL, __r8, ph2_ir->src0)); - emit(__movt(__AL, __r8, ph2_ir->src0)); + interm = mop->op == OP_address_of ? __sp : __r12; + if (mop->src0 > 255) { + emit(__movw(__AL, __r8, mop->src0)); + emit(__movt(__AL, __r8, mop->src0)); emit(__add_r(__AL, rd, interm, __r8)); } else - emit(__add_i(__AL, rd, interm, ph2_ir->src0)); + emit(__add_i(__AL, rd, interm, mop->src0)); return; case OP_assign: /* update_elf_offset() reserves no space for a self-assignment, so * emitting one here would push every later address out by four bytes * and leave the data segment's p_offset and p_vaddr incongruent. */ - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - rd == ph2_ir->src0_hi) { + if (MOP_PAIR_DEST_SRC0(mop) && rd == mop->src0_hi) { /* A pair moved one register over, as a slot round trip collapsed * into a move can leave it: the low destination is the high source, * so move the high word first, through r8 when the two registers * trade places. */ - if (ph2_ir->dest_hi == rn) { + if (mop->dest_hi == rn) { emit(__mov_r(__AL, __r8, rn)); - emit(__mov_r(__AL, rd, ph2_ir->src0_hi)); - emit(__mov_r(__AL, ph2_ir->dest_hi, __r8)); + emit(__mov_r(__AL, rd, mop->src0_hi)); + emit(__mov_r(__AL, mop->dest_hi, __r8)); } else { - emit(__mov_r(__AL, ph2_ir->dest_hi, ph2_ir->src0_hi)); + emit(__mov_r(__AL, mop->dest_hi, mop->src0_hi)); emit(__mov_r(__AL, rd, rn)); } return; } if (rd != rn) emit(__mov_r(__AL, rd, rn)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->dest_hi != ph2_ir->src0_hi) - emit(__mov_r(__AL, ph2_ir->dest_hi, ph2_ir->src0_hi)); + if (MOP_PAIR_DEST_SRC0(mop) && mop->dest_hi != mop->src0_hi) + emit(__mov_r(__AL, mop->dest_hi, mop->src0_hi)); return; case OP_load: case OP_global_load: - interm = ph2_ir->op == OP_load ? __sp : __r12; - if (ph2_ir->dest_hi >= 0) { - if (arm_offset_needs_movw(ph2_ir)) { - emit(__movw(__AL, __r8, ph2_ir->src0)); - emit(__movt(__AL, __r8, ph2_ir->src0)); + interm = mop->op == OP_load ? __sp : __r12; + if (mop->dest_hi >= 0) { + if (arm_offset_needs_movw(mop)) { + emit(__movw(__AL, __r8, mop->src0)); + emit(__movt(__AL, __r8, mop->src0)); emit(__add_r(__AL, __r8, interm, __r8)); emit(__lw(__AL, rd, __r8, 0)); - emit(__lw(__AL, ph2_ir->dest_hi, __r8, 4)); + emit(__lw(__AL, mop->dest_hi, __r8, 4)); } else { - emit(__lw(__AL, rd, interm, ph2_ir->src0)); - emit(__lw(__AL, ph2_ir->dest_hi, interm, ph2_ir->src0 + 4)); + emit(__lw(__AL, rd, interm, mop->src0)); + emit(__lw(__AL, mop->dest_hi, interm, mop->src0 + 4)); } return; } @@ -501,41 +244,41 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) /* LDRSB, LDRH and LDRSH have only eight immediate bits, unlike LDRB's * 12-bit field. Materialize a larger offset before loading. */ - if (arm_offset_needs_movw(ph2_ir)) { - emit(__movw(__AL, __r8, ph2_ir->src0)); - emit(__movt(__AL, __r8, ph2_ir->src0)); + if (arm_offset_needs_movw(mop)) { + emit(__movw(__AL, __r8, mop->src0)); + emit(__movt(__AL, __r8, mop->src0)); emit(__add_r(__AL, __r8, interm, __r8)); - if (ph2_ir->size_bytes == 1) - emit(ph2_ir->is_unsigned ? __lb(__AL, rd, __r8, 0) - : __lsb(__AL, rd, __r8, 0)); - else if (ph2_ir->size_bytes == 2) - emit(ph2_ir->is_unsigned ? __lhu(__AL, rd, __r8, 0) - : __lh(__AL, rd, __r8, 0)); + if (mop->size_bytes == 1) + emit(mop->is_unsigned ? __lb(__AL, rd, __r8, 0) + : __lsb(__AL, rd, __r8, 0)); + else if (mop->size_bytes == 2) + emit(mop->is_unsigned ? __lhu(__AL, rd, __r8, 0) + : __lh(__AL, rd, __r8, 0)); else emit(__lw(__AL, rd, __r8, 0)); - } else if (ph2_ir->size_bytes == 1) - emit(ph2_ir->is_unsigned ? __lb(__AL, rd, interm, ph2_ir->src0) - : __lsb(__AL, rd, interm, ph2_ir->src0)); - else if (ph2_ir->size_bytes == 2) - emit(ph2_ir->is_unsigned ? __lhu(__AL, rd, interm, ph2_ir->src0) - : __lh(__AL, rd, interm, ph2_ir->src0)); + } else if (mop->size_bytes == 1) + emit(mop->is_unsigned ? __lb(__AL, rd, interm, mop->src0) + : __lsb(__AL, rd, interm, mop->src0)); + else if (mop->size_bytes == 2) + emit(mop->is_unsigned ? __lhu(__AL, rd, interm, mop->src0) + : __lh(__AL, rd, interm, mop->src0)); else - emit(__lw(__AL, rd, interm, ph2_ir->src0)); + emit(__lw(__AL, rd, interm, mop->src0)); return; case OP_store: case OP_global_store: - interm = ph2_ir->op == OP_store ? __sp : __r12; - store_offset = ph2_ir->src1; - if (ph2_ir->src0_hi >= 0) { - if (arm_offset_needs_movw(ph2_ir)) { + interm = mop->op == OP_store ? __sp : __r12; + store_offset = mop->src1; + if (mop->src0_hi >= 0) { + if (arm_offset_needs_movw(mop)) { emit(__movw(__AL, __r8, store_offset)); emit(__movt(__AL, __r8, store_offset)); emit(__add_r(__AL, __r8, interm, __r8)); emit(__sw(__AL, rn, __r8, 0)); - emit(__sw(__AL, ph2_ir->src0_hi, __r8, 4)); + emit(__sw(__AL, mop->src0_hi, __r8, 4)); } else { emit(__sw(__AL, rn, interm, store_offset)); - emit(__sw(__AL, ph2_ir->src0_hi, interm, store_offset + 4)); + emit(__sw(__AL, mop->src0_hi, interm, store_offset + 4)); } return; } @@ -543,7 +286,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) /* STRH has an 8-bit split immediate while STR/STRB accept 12 bits. * Materialize larger halfword offsets before selecting the width. */ - if (arm_offset_needs_movw(ph2_ir)) { + if (arm_offset_needs_movw(mop)) { emit(__movw(__AL, __r8, store_offset)); emit(__movt(__AL, __r8, store_offset)); emit(__add_r(__AL, __r8, interm, __r8)); @@ -556,93 +299,79 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) * neighbouring designated members on ARM. Ordinary stack slots remain * word-sized unless their IR says otherwise. */ - if (ph2_ir->size_bytes == 1) + if (mop->size_bytes == 1) emit(__sb(__AL, rn, interm, store_offset)); - else if (ph2_ir->size_bytes == 2) + else if (mop->size_bytes == 2) emit(__sh(__AL, rn, interm, store_offset)); else emit(__sw(__AL, rn, interm, store_offset)); return; case OP_read: - if (ph2_ir->dest_hi >= 0) { - /* A pair read through a pointer. When the low destination is the - * address register itself, fetch the high word first so the address - * survives until both words are loaded. - */ - if (ph2_ir->src1 != 8) + if (mop->dest_hi >= 0) { + if (mop->src1 != 8) fatal("unsupported Arm pair load width"); - if (rd == rn) { - emit(__lw(__AL, ph2_ir->dest_hi, rn, 4)); - emit(__lw(__AL, rd, rn, 0)); - } else { - emit(__lw(__AL, rd, rn, 0)); - emit(__lw(__AL, ph2_ir->dest_hi, rn, 4)); - } +#define ARM_WORD_LOAD(dst, base, offset) emit(__lw(__AL, dst, base, offset)) + EMIT_PAIR_LOAD(ARM_WORD_LOAD, rd, mop->dest_hi, rn); +#undef ARM_WORD_LOAD return; } - if (ph2_ir->src1 == 1) - emit(ph2_ir->is_unsigned ? __lb(__AL, rd, rn, 0) - : __lsb(__AL, rd, rn, 0)); - else if (ph2_ir->src1 == 2) - emit(ph2_ir->is_unsigned ? __lhu(__AL, rd, rn, 0) - : __lh(__AL, rd, rn, 0)); - else if (ph2_ir->src1 == 4) + if (mop->src1 == 1) + emit(mop->is_unsigned ? __lb(__AL, rd, rn, 0) + : __lsb(__AL, rd, rn, 0)); + else if (mop->src1 == 2) + emit(mop->is_unsigned ? __lhu(__AL, rd, rn, 0) + : __lh(__AL, rd, rn, 0)); + else if (mop->src1 == 4) emit(__lw(__AL, rd, rn, 0)); else fatal("unsupported Arm load width"); return; case OP_write: - if (ph2_ir->src1_hi >= 0) { - if (ph2_ir->dest != 8) + if (mop->src1_hi >= 0) { + if (mop->dest != 8) fatal("unsupported Arm pair store width"); emit(__sw(__AL, rm, rn, 0)); - emit(__sw(__AL, ph2_ir->src1_hi, rn, 4)); + emit(__sw(__AL, mop->src1_hi, rn, 4)); return; } - if (ph2_ir->dest == 1) + if (mop->dest == 1) emit(__sb(__AL, rm, rn, 0)); - else if (ph2_ir->dest == 2) + else if (mop->dest == 2) emit(__sh(__AL, rm, rn, 0)); - else if (ph2_ir->dest == 4) + else if (mop->dest == 4) emit(__sw(__AL, rm, rn, 0)); else fatal("unsupported Arm store width"); return; case OP_branch: /* A pair is true when either word is nonzero. */ - if (ph2_ir->src0_hi >= 0) { - emit(__or_r(__AL, __r8, rn, ph2_ir->src0_hi)); + if (mop->src0_hi >= 0) { + emit(__or_r(__AL, __r8, rn, mop->src0_hi)); emit(__teq(__r8)); } else emit(__teq(rn)); - if (ph2_ir->is_branch_detached) { + if (mop->src2 == 2) + /* The then block follows: only the else edge needs a branch. */ + emit(__b(__EQ, mop->else_bb->elf_offset - elf_code->size)); + else if (!mop->src2) { /* The else block does not follow, and nothing says the then block * does either: a loop's back edge lands behind this one. Jump to * both explicitly rather than skipping over the else jump into - * whatever was laid out next. The estimator charges 12 bytes for - * this form either way. + * whatever was laid out next. */ - emit(__b(__NE, ph2_ir->then_bb->elf_offset - elf_code->size)); - emit(__b(__AL, ph2_ir->else_bb->elf_offset - elf_code->size)); + emit(__b(__NE, mop->then_bb->elf_offset - elf_code->size)); + emit(__b(__AL, mop->else_bb->elf_offset - elf_code->size)); } else - emit(__b(__NE, ph2_ir->then_bb->elf_offset - elf_code->size)); + emit(__b(__NE, mop->then_bb->elf_offset - elf_code->size)); return; case OP_jump: - emit(__b(__AL, ph2_ir->next_bb->elf_offset - elf_code->size)); + if (!mop->src2) + emit(__b(__AL, mop->next_bb->elf_offset - elf_code->size)); return; case OP_call: - func = find_func(ph2_ir->func_name); - if (func->bbs) - ofs = func->bbs->elf_offset - elf_code->size; - else if (dynlink) { - ofs = (dynamic_sections.elf_plt_start + func->plt_offset) - - (elf_code_start + elf_code->size); - is_external_call = true; - } else { - printf("The '%s' function is not implemented\n", ph2_ir->func_name); - fflush(stdout); /* see fatal() */ - abort(); - } + ofs = function_entry_address(mop->func_name) - + (elf_code_start + elf_code->size); + is_external_call = !find_func(mop->func_name)->bbs; /* When calling external functions in dynamic linking mode, the * following instructions are required: @@ -667,24 +396,15 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) } return; case OP_load_data_address: - emit(__movw(__AL, rd, ph2_ir->src0 + elf_data_start)); - emit(__movt(__AL, rd, ph2_ir->src0 + elf_data_start)); + emit(__movw(__AL, rd, mop->src0 + elf_data_start)); + emit(__movt(__AL, rd, mop->src0 + elf_data_start)); return; case OP_load_rodata_address: - emit(__movw(__AL, rd, ph2_ir->src0 + elf_rodata_start)); - emit(__movt(__AL, rd, ph2_ir->src0 + elf_rodata_start)); + emit(__movw(__AL, rd, mop->src0 + elf_rodata_start)); + emit(__movt(__AL, rd, mop->src0 + elf_rodata_start)); return; case OP_address_of_func: - func = find_func(ph2_ir->func_name); - if (func->bbs) - ofs = elf_code_start + func->bbs->elf_offset; - else if (dynlink) - ofs = dynamic_sections.elf_plt_start + func->plt_offset; - else { - printf("The '%s' function is not implemented\n", ph2_ir->func_name); - fflush(stdout); /* see fatal() */ - abort(); - } + ofs = function_entry_address(mop->func_name); emit(__movw(__AL, __r8, ofs)); emit(__movt(__AL, __r8, ofs)); emit(__sw(__AL, __r8, rn, 0)); @@ -706,12 +426,12 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) } return; case OP_return: - if (ph2_ir->src0 == -1) + if (mop->src0 == -1) emit(__mov_r(__AL, __r0, __r0)); else emit(__mov_r(__AL, __r0, rn)); - if (ph2_ir->src0_hi >= 0) - emit(__mov_r(__AL, __r1, ph2_ir->src0_hi)); + if (mop->src0_hi >= 0) + emit(__mov_r(__AL, __r1, mop->src0_hi)); /* When calling a function, the following operations are performed: * 1. push r4-r11 and lr onto the stack. @@ -722,7 +442,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) * upon returning to restore the stack and the contents of r4-r11 and * lr. */ - ofs = ALIGN_UP(ph2_ir->src1, MIN_ALIGNMENT) + 4; + ofs = ALIGN_UP(mop->src1, MIN_ALIGNMENT) + 4; emit(__movw(__AL, __r8, ofs)); emit(__movt(__AL, __r8, ofs)); emit(__add_r(__AL, __sp, __sp, __r8)); @@ -730,45 +450,38 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) emit(__bx(__AL, __lr)); return; case OP_add: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) { + if (MOP_PAIR_BINARY(mop)) { emit(__adds_r(__AL, rd, rn, rm)); - emit(__adc_r(__AL, ph2_ir->dest_hi, ph2_ir->src0_hi, - ph2_ir->src1_hi)); + emit(__adc_r(__AL, mop->dest_hi, mop->src0_hi, mop->src1_hi)); } else emit(__add_r(__AL, rd, rn, rm)); return; case OP_sub: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) { + if (MOP_PAIR_BINARY(mop)) { emit(__subs_r(__AL, rd, rn, rm)); - emit(__sbc_r(__AL, ph2_ir->dest_hi, ph2_ir->src0_hi, - ph2_ir->src1_hi)); + emit(__sbc_r(__AL, mop->dest_hi, mop->src0_hi, mop->src1_hi)); } else emit(__sub_r(__AL, rd, rn, rm)); return; case OP_mul: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) { + if (MOP_PAIR_BINARY(mop)) { /* Sum the cross terms before UMULL writes the result pair, so the * product stays right even if that pair shares a register with an * operand. The allocator does not hand out such a pair today, and * this order costs nothing over the other. */ - emit(__mul(__AL, __r8, rn, ph2_ir->src1_hi)); - emit(__mul(__AL, __r9, ph2_ir->src0_hi, rm)); + emit(__mul(__AL, __r8, rn, mop->src1_hi)); + emit(__mul(__AL, __r9, mop->src0_hi, rm)); emit(__add_r(__AL, __r8, __r8, __r9)); - emit(__umull(__AL, rd, ph2_ir->dest_hi, rn, rm)); - emit(__add_r(__AL, ph2_ir->dest_hi, ph2_ir->dest_hi, __r8)); + emit(__umull(__AL, rd, mop->dest_hi, rn, rm)); + emit(__add_r(__AL, mop->dest_hi, mop->dest_hi, __r8)); } else emit(__mul(__AL, rd, rn, rm)); return; case OP_div: case OP_mod: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) { - bool is_unsigned = - ph2_ir->src0_is_unsigned || ph2_ir->src1_is_unsigned; + if (MOP_PAIR_BINARY(mop)) { + bool is_unsigned = mop->src0_is_unsigned || mop->src1_is_unsigned; /* Stack words: dividend (0,4), divisor (8,12), remainder (16,20), * quotient (24,28), counter (32), and the incoming dividend bit @@ -777,9 +490,9 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) */ emit(__add_i(__AL, __sp, __sp, is_unsigned ? -40 : -48)); emit(__sw(__AL, rn, __sp, 0)); - emit(__sw(__AL, ph2_ir->src0_hi, __sp, 4)); + emit(__sw(__AL, mop->src0_hi, __sp, 4)); emit(__sw(__AL, rm, __sp, 8)); - emit(__sw(__AL, ph2_ir->src1_hi, __sp, 12)); + emit(__sw(__AL, mop->src1_hi, __sp, 12)); if (!is_unsigned) { /* x ^ sign + -sign produces its unsigned magnitude. */ emit(__lw(__AL, __r8, __sp, 4)); @@ -877,32 +590,31 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) emit(__cmp_i(__AL, __r8, 0)); emit(__b(__NE, -220)); - emit(__lw(__AL, rd, __sp, ph2_ir->op == OP_mod ? 16 : 24)); - emit(__lw(__AL, ph2_ir->dest_hi, __sp, - ph2_ir->op == OP_mod ? 20 : 28)); + emit(__lw(__AL, rd, __sp, mop->op == OP_mod ? 16 : 24)); + emit(__lw(__AL, mop->dest_hi, __sp, mop->op == OP_mod ? 20 : 28)); if (!is_unsigned) { emit(__lw(__AL, __r10, __sp, 40)); - if (ph2_ir->op == OP_div) { + if (mop->op == OP_div) { emit(__lw(__AL, __r8, __sp, 44)); emit(__eor_r(__AL, __r10, __r10, __r8)); } emit(__eor_r(__AL, rd, rd, __r10)); - emit(__eor_r(__AL, ph2_ir->dest_hi, ph2_ir->dest_hi, __r10)); + emit(__eor_r(__AL, mop->dest_hi, mop->dest_hi, __r10)); emit(__rsbs_i(__AL, __r8, 0, __r10)); emit(__adds_r(__AL, rd, rd, __r8)); emit(__zero(__r8)); - emit(__adc_r(__AL, ph2_ir->dest_hi, ph2_ir->dest_hi, __r8)); + emit(__adc_r(__AL, mop->dest_hi, mop->dest_hi, __r8)); } emit(__add_i(__AL, __sp, __sp, is_unsigned ? 40 : 48)); return; } if (hard_mul_div) { - if (ph2_ir->op == OP_div) - emit(ph2_ir->src0_is_unsigned || ph2_ir->src1_is_unsigned + if (mop->op == OP_div) + emit(mop->src0_is_unsigned || mop->src1_is_unsigned ? __udiv(__AL, rd, rm, rn) : __div(__AL, rd, rm, rn)); else { - emit(ph2_ir->src0_is_unsigned || ph2_ir->src1_is_unsigned + emit(mop->src0_is_unsigned || mop->src1_is_unsigned ? __udiv(__AL, __r8, rm, rn) : __div(__AL, __r8, rm, rn)); emit(__mul(__AL, __r8, rm, __r8)); @@ -917,7 +629,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) /* Unsigned operands already are magnitudes. Keep the signed path's * instruction count so the fixed branch displacements remain valid. */ - if (ph2_ir->src0_is_unsigned || ph2_ir->src1_is_unsigned) { + if (mop->src0_is_unsigned || mop->src1_is_unsigned) { emit(__zero(__r8)); emit(__mov_r(__AL, __r8, __r8)); emit(__mov_r(__AL, __r8, __r8)); @@ -932,7 +644,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) emit(__add_r(__AL, rm, rm, __r9)); emit(__eor_r(__AL, rm, rm, __r9)); } - if (ph2_ir->op == OP_div) + if (mop->op == OP_div) emit(__eor_r(__AL, __r10, __r8, __r9)); else { /* If the requested operation is modulo, the result will be stored @@ -984,30 +696,30 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) emit(__rsb_i(__NE, rd, 0, rd)); return; case OP_lshift: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) { + if (MOP_PAIR_DEST_SRC0(mop)) { emit(__cmp_i(__AL, rm, 32)); emit(__b(__CS, 32)); emit(__mov_r(__AL, __r8, rn)); emit(__sll(__AL, rd, rn, rm)); - emit(__sll(__AL, ph2_ir->dest_hi, ph2_ir->src0_hi, rm)); + emit(__sll(__AL, mop->dest_hi, mop->src0_hi, rm)); emit(__rsb_i(__AL, __r9, 32, rm)); emit(__srl(__AL, __r8, __r8, __r9)); - emit(__or_r(__AL, ph2_ir->dest_hi, ph2_ir->dest_hi, __r8)); + emit(__or_r(__AL, mop->dest_hi, mop->dest_hi, __r8)); emit(__b(__AL, 16)); emit(__add_i(__AL, __r9, rm, -32)); - emit(__sll(__AL, ph2_ir->dest_hi, rn, __r9)); + emit(__sll(__AL, mop->dest_hi, rn, __r9)); emit(__zero(rd)); return; } emit(__sll(__AL, rd, rn, rm)); return; case OP_rshift: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) { - int shift_kind = ph2_ir->src0_is_unsigned ? logic_rs : arith_rs; + if (MOP_PAIR_DEST_SRC0(mop)) { + int shift_kind = mop->src0_is_unsigned ? logic_rs : arith_rs; emit(__cmp_i(__AL, rm, 32)); emit(__b(__CS, 32)); - emit(__mov_r(__AL, __r8, ph2_ir->src0_hi)); + emit(__mov_r(__AL, __r8, mop->src0_hi)); /* The sign lives in the high word. The low word's vacated top bits * are the high word's low bits, ORed in below, so shifting in the @@ -1018,22 +730,21 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) emit(__sll(__AL, __r8, __r8, __r9)); emit(__or_r(__AL, rd, rd, __r8)); emit(shift_kind == logic_rs - ? __srl(__AL, ph2_ir->dest_hi, ph2_ir->src0_hi, rm) - : __sra(__AL, ph2_ir->dest_hi, ph2_ir->src0_hi, rm)); + ? __srl(__AL, mop->dest_hi, mop->src0_hi, rm) + : __sra(__AL, mop->dest_hi, mop->src0_hi, rm)); emit(__b(__AL, 16)); emit(__add_i(__AL, __r9, rm, -32)); - emit(shift_kind == logic_rs - ? __srl(__AL, rd, ph2_ir->src0_hi, __r9) - : __sra(__AL, rd, ph2_ir->src0_hi, __r9)); + emit(shift_kind == logic_rs ? __srl(__AL, rd, mop->src0_hi, __r9) + : __sra(__AL, rd, mop->src0_hi, __r9)); if (shift_kind == logic_rs) - emit(__zero(ph2_ir->dest_hi)); + emit(__zero(mop->dest_hi)); else - emit(__srl_amt(__AL, 0, arith_rs, ph2_ir->dest_hi, - ph2_ir->src0_hi, 31)); + emit(__srl_amt(__AL, 0, arith_rs, mop->dest_hi, mop->src0_hi, + 31)); return; } - emit(ph2_ir->src0_is_unsigned ? __srl(__AL, rd, rn, rm) - : __sra(__AL, rd, rn, rm)); + emit(mop->src0_is_unsigned ? __srl(__AL, rd, rn, rm) + : __sra(__AL, rd, rn, rm)); return; case OP_eq: case OP_neq: @@ -1050,67 +761,53 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) * any operand register. Equality is left to the word-by-word sequence * below, since Z is not valid after SBCS. */ - if (ph2_ir->op != OP_eq && ph2_ir->op != OP_neq && - ph2_ir->src0_hi >= 0 && ph2_ir->src1_hi >= 0) { - bool unsigned_cmp = - ph2_ir->src0_is_unsigned || ph2_ir->src1_is_unsigned; - bool swap = ph2_ir->op == OP_gt || ph2_ir->op == OP_leq; - bool below = ph2_ir->op == OP_lt || ph2_ir->op == OP_gt; + if (mop->op != OP_eq && mop->op != OP_neq && mop->src0_hi >= 0 && + mop->src1_hi >= 0) { + bool unsigned_cmp = mop->src0_is_unsigned || mop->src1_is_unsigned; + bool swap = mop->op == OP_gt || mop->op == OP_leq; + bool below = mop->op == OP_lt || mop->op == OP_gt; arm_cond_t cond = below ? (unsigned_cmp ? __CC : __LT) : (unsigned_cmp ? __CS : __GE); emit(__cmp_r(__AL, swap ? rm : rn, swap ? rn : rm)); - emit(__sbcs_r(__AL, __r8, swap ? ph2_ir->src1_hi : ph2_ir->src0_hi, - swap ? ph2_ir->src0_hi : ph2_ir->src1_hi)); + emit(__sbcs_r(__AL, __r8, swap ? mop->src1_hi : mop->src0_hi, + swap ? mop->src0_hi : mop->src1_hi)); emit(__zero(rd)); emit(__mov_i(cond, rd, 1)); return; } emit(__cmp_r(__AL, rn, rm)); - if ((ph2_ir->op == OP_eq || ph2_ir->op == OP_neq) && - ph2_ir->src0_hi >= 0 && ph2_ir->src1_hi >= 0) - emit(__cmp_r(__EQ, ph2_ir->src0_hi, ph2_ir->src1_hi)); + if ((mop->op == OP_eq || mop->op == OP_neq) && mop->src0_hi >= 0 && + mop->src1_hi >= 0) + emit(__cmp_r(__EQ, mop->src0_hi, mop->src1_hi)); emit(__zero(rd)); - emit(__mov_i(arm_get_cond(ph2_ir->op, ph2_ir->src0_is_unsigned || - ph2_ir->src1_is_unsigned), + emit(__mov_i(arm_get_cond(mop->op, mop->src0_is_unsigned || + mop->src1_is_unsigned), rd, 1)); return; case OP_negate: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) { + if (MOP_PAIR_DEST_SRC0(mop)) { emit(__rsbs_i(__AL, rd, 0, rn)); - emit(__rsc_i(__AL, ph2_ir->dest_hi, 0, ph2_ir->src0_hi)); + emit(__rsc_i(__AL, mop->dest_hi, 0, mop->src0_hi)); } else emit(__rsb_i(__AL, rd, 0, rn)); return; case OP_bit_not: emit(__mvn_r(__AL, rd, rn)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) - emit(__mvn_r(__AL, ph2_ir->dest_hi, ph2_ir->src0_hi)); + if (MOP_PAIR_DEST_SRC0(mop)) + emit(__mvn_r(__AL, mop->dest_hi, mop->src0_hi)); return; case OP_bit_and: - emit(__and_r(__AL, rd, rn, rm)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - emit(__and_r(__AL, ph2_ir->dest_hi, ph2_ir->src0_hi, - ph2_ir->src1_hi)); - return; case OP_bit_or: - emit(__or_r(__AL, rd, rn, rm)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - emit(__or_r(__AL, ph2_ir->dest_hi, ph2_ir->src0_hi, - ph2_ir->src1_hi)); - return; case OP_bit_xor: - emit(__eor_r(__AL, rd, rn, rm)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - emit(__eor_r(__AL, ph2_ir->dest_hi, ph2_ir->src0_hi, - ph2_ir->src1_hi)); + emit(arm_bitwise_insn(mop->op, rd, rn, rm)); + if (MOP_PAIR_BINARY(mop)) + emit(arm_bitwise_insn(mop->op, mop->dest_hi, mop->src0_hi, + mop->src1_hi)); return; case OP_log_not: - if (ph2_ir->src0_hi >= 0) { - emit(__or_r(__AL, __r8, rn, ph2_ir->src0_hi)); + if (mop->src0_hi >= 0) { + emit(__or_r(__AL, __r8, rn, mop->src0_hi)); emit(__cmp_i(__AL, __r8, 0)); } else emit(__cmp_i(__AL, rn, 0)); @@ -1118,7 +815,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) emit(__mov_i(__EQ, rd, 1)); return; case OP_trunc: - if (ph2_ir->is_unsigned && (rm == 1 || rm == 2)) { + if (mop->is_unsigned && (rm == 1 || rm == 2)) { int shift = rm == 1 ? 24 : 16; emit(__sll_amt(__AL, 0, logic_ls, rd, rn, shift)); @@ -1136,28 +833,28 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) case OP_sign_ext: { /* Decode source size from upper 16 bits */ int source_size = (rm >> 16) & 0xFFFF; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi < 0) { + if (mop->dest_hi >= 0 && mop->src0_hi < 0) { if (source_size == 1) { - if (ph2_ir->src0_is_unsigned || ph2_ir->src0_is_pointer) { + if (mop->src0_is_unsigned || mop->src0_is_pointer) { emit(__sll_amt(__AL, 0, logic_ls, rd, rn, 24)); emit(__srl_amt(__AL, 0, logic_rs, rd, rd, 24)); } else emit(__sxtb(__AL, rd, rn, 0)); } else if (source_size == 2) { - if (ph2_ir->src0_is_unsigned || ph2_ir->src0_is_pointer) { + if (mop->src0_is_unsigned || mop->src0_is_pointer) { emit(__sll_amt(__AL, 0, logic_ls, rd, rn, 16)); emit(__srl_amt(__AL, 0, logic_rs, rd, rd, 16)); } else emit(__sxth(__AL, rd, rn, 0)); } else emit(__mov_r(__AL, rd, rn)); - if (ph2_ir->src0_is_unsigned || ph2_ir->src0_is_pointer) - emit(__zero(ph2_ir->dest_hi)); + if (mop->src0_is_unsigned || mop->src0_is_pointer) + emit(__zero(mop->dest_hi)); else - emit(__srl_amt(__AL, 0, arith_rs, ph2_ir->dest_hi, rd, 31)); + emit(__srl_amt(__AL, 0, arith_rs, mop->dest_hi, rd, 31)); return; } - if (ph2_ir->src0_is_unsigned) { + if (mop->src0_is_unsigned) { int shift = source_size == 2 ? 16 : 24; emit(__sll_amt(__AL, 0, logic_ls, rd, rn, shift)); emit(__srl_amt(__AL, 0, logic_rs, rd, rd, shift)); @@ -1177,14 +874,14 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) * zero-extend the source according to its original type. */ emit(__mov_r(__AL, rd, rn)); - if (ph2_ir->dest_hi >= 0) { - if (ph2_ir->src0_hi >= 0) { - if (ph2_ir->dest_hi != ph2_ir->src0_hi) - emit(__mov_r(__AL, ph2_ir->dest_hi, ph2_ir->src0_hi)); - } else if (ph2_ir->src0_is_unsigned || ph2_ir->src0_is_pointer) - emit(__zero(ph2_ir->dest_hi)); + if (mop->dest_hi >= 0) { + if (mop->src0_hi >= 0) { + if (mop->dest_hi != mop->src0_hi) + emit(__mov_r(__AL, mop->dest_hi, mop->src0_hi)); + } else if (mop->src0_is_unsigned || mop->src0_is_pointer) + emit(__zero(mop->dest_hi)); else - emit(__srl_amt(__AL, 0, arith_rs, ph2_ir->dest_hi, rn, 31)); + emit(__srl_amt(__AL, 0, arith_rs, mop->dest_hi, rn, 31)); } return; default: @@ -1340,10 +1037,7 @@ void code_generate(void) emit(__bx(__AL, __lr)); } - ph2_ir_t *ph2_ir; - for (ph2_ir = GLOBAL_FUNC->bbs->ph2_ir_list.head; ph2_ir; - ph2_ir = ph2_ir->next) - emit_ph2_ir(ph2_ir); + mop_visit_globals(emit_mop); /* prepare 'argc' and 'argv', then proceed to 'main' function */ if (MAIN_BB) { @@ -1381,9 +1075,9 @@ void code_generate(void) } } - for (int i = 0; i < ph2_ir_idx; i++) { - ph2_ir = PH2_IR_FLATTEN[i]; - emit_ph2_ir(ph2_ir); + for (int i = 0; i < mop_flat_count; i++) { + vir_mop_t *mop = MOP_FLAT[i]; + emit_mop(mop); } } diff --git a/src/arm.c b/src/arm.c index ee101317..e9957549 100644 --- a/src/arm.c +++ b/src/arm.c @@ -126,7 +126,9 @@ int arm_extract_bits(int imm, int i_start, int i_end, int d_start, int d_end) int arm_encode(arm_cond_t cond, int opcode, int rn, int rd, int op2) { - return (cond << 28) + (opcode << 20) + (rn << 16) + (rd << 12) + op2; + /* The condition fills bits 31:28, which a signed shift cannot reach. */ + return ((unsigned int) cond << 28) + (opcode << 20) + (rn << 16) + + (rd << 12) + op2; } /* ARM A8.8.247: unsigned 32x32 -> 64 multiply. RdLo/RdHi receive the complete @@ -134,8 +136,8 @@ int arm_encode(arm_cond_t cond, int opcode, int rn, int rd, int op2) */ int __umull(arm_cond_t cond, arm_reg rdlo, arm_reg rdhi, arm_reg rm, arm_reg rs) { - return (cond << 28) + 0x00800090 + (rdhi << 16) + (rdlo << 12) + (rs << 8) + - rm; + return ((unsigned int) cond << 28) + 0x00800090 + (rdhi << 16) + + (rdlo << 12) + (rs << 8) + rm; } int __svc(void) @@ -167,20 +169,15 @@ int __mov(arm_cond_t cond, int io, int opcode, int s, int rn, int rd, int op2) (shift << 8) + (encoded_op2 & 255)); } -int __and_r(arm_cond_t cond, arm_reg rd, arm_reg rs, arm_reg rm) -{ - return __mov(cond, 0, arm_and, 0, rs, rd, rm); -} - -int __or_r(arm_cond_t cond, arm_reg rd, arm_reg rs, arm_reg rm) -{ - return __mov(cond, 0, arm_orr, 0, rs, rd, rm); -} - -int __eor_r(arm_cond_t cond, arm_reg rd, arm_reg rs, arm_reg rm) -{ - return __mov(cond, 0, arm_eor, 0, rs, rd, rm); -} +/* Preserve each register ALU helper's opcode, flag update and operand name. */ +#define ARM_REGISTER_INSN(name, opcode, flags, operand) \ + int name(arm_cond_t cond, arm_reg rd, arm_reg rs, arm_reg operand) \ + { \ + return __mov(cond, 0, opcode, flags, rs, rd, operand); \ + } +ARM_REGISTER_INSN(__and_r, arm_and, 0, rm) +ARM_REGISTER_INSN(__or_r, arm_orr, 0, rm) +ARM_REGISTER_INSN(__eor_r, arm_eor, 0, rm) int __mvn_r(arm_cond_t cond, arm_reg rd, arm_reg rm) { @@ -218,16 +215,16 @@ int __srl(arm_cond_t cond, arm_reg rd, arm_reg rm, arm_reg rs) rm + (1 << 4) + (1 << 5) + (rs << 8)); } -int __srl_amt(arm_cond_t cond, - int s, - shift_type shift, - arm_reg rd, - arm_reg rm, - int amt) -{ - return arm_encode(cond, s + (arm_mov << 1) + (0 << 5), 0, rd, - rm + (0 << 4) + (shift << 5) + (amt << 7)); -} +#define DEFINE_SHIFT_IMMEDIATE(name) \ + int name(arm_cond_t cond, int s, shift_type shift, arm_reg rd, arm_reg rm, \ + int amt) \ + { \ + return arm_encode(cond, s + (arm_mov << 1) + (0 << 5), 0, rd, \ + rm + (0 << 4) + (shift << 5) + (amt << 7)); \ + } +DEFINE_SHIFT_IMMEDIATE(__srl_amt) +DEFINE_SHIFT_IMMEDIATE(__sll_amt) +#undef DEFINE_SHIFT_IMMEDIATE int __sll(arm_cond_t cond, arm_reg rd, arm_reg rm, arm_reg rs) { @@ -235,17 +232,6 @@ int __sll(arm_cond_t cond, arm_reg rd, arm_reg rm, arm_reg rs) rm + (1 << 4) + (0 << 5) + (rs << 8)); } -int __sll_amt(arm_cond_t cond, - int s, - shift_type shift, - arm_reg rd, - arm_reg rm, - int amt) -{ - return arm_encode(cond, s + (arm_mov << 1) + (0 << 5), 0, rd, - rm + (0 << 4) + (shift << 5) + (amt << 7)); -} - int __sra(arm_cond_t cond, arm_reg rd, arm_reg rm, arm_reg rs) { /* Arithmetic right shift with register Bit 4 = 1 (register-specified shift) @@ -261,41 +247,13 @@ int __add_i(arm_cond_t cond, arm_reg rd, arm_reg rs, int imm) return __mov(cond, 1, arm_add, 0, rs, rd, imm); return __mov(cond, 1, arm_sub, 0, rs, rd, -imm); } - -int __add_r(arm_cond_t cond, arm_reg rd, arm_reg rs, arm_reg ro) -{ - return __mov(cond, 0, arm_add, 0, rs, rd, ro); -} - -int __adds_r(arm_cond_t cond, arm_reg rd, arm_reg rs, arm_reg ro) -{ - return __mov(cond, 0, arm_add, 1, rs, rd, ro); -} - -int __adc_r(arm_cond_t cond, arm_reg rd, arm_reg rs, arm_reg ro) -{ - return __mov(cond, 0, arm_adc, 0, rs, rd, ro); -} - -int __sub_r(arm_cond_t cond, arm_reg rd, arm_reg rs, arm_reg ro) -{ - return __mov(cond, 0, arm_sub, 0, rs, rd, ro); -} - -int __subs_r(arm_cond_t cond, arm_reg rd, arm_reg rs, arm_reg ro) -{ - return __mov(cond, 0, arm_sub, 1, rs, rd, ro); -} - -int __sbc_r(arm_cond_t cond, arm_reg rd, arm_reg rs, arm_reg ro) -{ - return __mov(cond, 0, arm_sbc, 0, rs, rd, ro); -} - -int __sbcs_r(arm_cond_t cond, arm_reg rd, arm_reg rs, arm_reg ro) -{ - return __mov(cond, 0, arm_sbc, 1, rs, rd, ro); -} +ARM_REGISTER_INSN(__add_r, arm_add, 0, ro) +ARM_REGISTER_INSN(__adds_r, arm_add, 1, ro) +ARM_REGISTER_INSN(__adc_r, arm_adc, 0, ro) +ARM_REGISTER_INSN(__sub_r, arm_sub, 0, ro) +ARM_REGISTER_INSN(__subs_r, arm_sub, 1, ro) +ARM_REGISTER_INSN(__sbc_r, arm_sbc, 0, ro) +ARM_REGISTER_INSN(__sbcs_r, arm_sbc, 1, ro) int __rsbs_i(arm_cond_t cond, arm_reg rd, int imm, arm_reg rn) { @@ -454,7 +412,7 @@ int __bl(arm_cond_t cond, int ofs) int __bx(arm_cond_t cond, arm_reg rm) { /* BX: Branch and Exchange */ - return (cond << 28) | 0x012FFF10 | rm; + return ((unsigned int) cond << 28) | 0x012FFF10 | rm; } int __blx(arm_cond_t cond, arm_reg rd) @@ -514,3 +472,5 @@ int __sxth(arm_cond_t cond, arm_reg rd, arm_reg rm, int rotation) return arm_encode(cond, 107, 0xF, rd, rm | ((rotation >> 3) << 10) | (0x7 << 4)); } + +#undef ARM_REGISTER_INSN diff --git a/src/arm64-codegen.c b/src/arm64-codegen.c index dcbbe435..f938b755 100644 --- a/src/arm64-codegen.c +++ b/src/arm64-codegen.c @@ -1,6 +1,7 @@ /* * AArch64 Linux code generator. The allocator's virtual registers map to - * x0..x7, x20..x22; x16/x17 are reserved scratch and x19 is the global base. + * x0..x7, x9..x15, x20..x28; x16/x17 are reserved scratch and x19 is the global + * base. */ #include "arm64.c" #include "defs.h" @@ -12,23 +13,153 @@ */ #define A64_GP 19 -/* AAPCS64 keeps SP 16-byte aligned, and the prologue saves five registers in - * three pairs. Every frame calculation derives from these two. +/* AAPCS64 keeps SP 16-byte aligned. Save used callee registers in pairs, + * together with FP/LR and the global base when dynamic main replaces it. */ #define A64_STACK_ALIGN 16 -#define A64_SAVE_BYTES 48 +#define A64_FIRST_CALLEE (REG_CNT - CALLEE_SAVED_REGS) +#define A64_SAVE_GP 16 + +/* A leaf with no frame, no saved register and no stack argument: it keeps + * neither FP nor LR, so it pushes nothing and returns straight to LR. + */ +#define A64_FRAMELESS 32 + +/* A return that only moves its value into x0 and branches to the first return's + * epilogue, at the code offset its dest holds, instead of repeating it. + */ +#define A64_SHARED_EXIT 64 +#define A64_SHARED_EXIT_WORDS 4 + +/* The last pair the prologue pushes also allocates the locals, which then sit + * above it, and the epilogue's first pop frees them again: one STP and one LDP + * with writeback instead of a separate SUB and ADD. LDP's writeback reaches 504 + * bytes, the pair's 16 and the locals together, and the locals must not start + * at SP when a call passes arguments there. + */ +#define A64_FOLDED_FRAME 128 +#define A64_FOLDED_FRAME_MAX 480 + +/* A small loop should not straddle a 64-byte fetch block: on the eMAG a + * five-instruction loop that did ran at four fifths of the speed. Function + * entries gain nothing measurable from alignment, and no loop is worth more + * than four no-ops of padding. + */ +#define A64_FETCH_BLOCK 64 +#define A64_LOOP_PADDING 16 + +/* Past the two scratch registers, the value a store writes when it stores zero: + * as the data register of a load or store, register 31 reads as zero. + */ +#define A64_ZERO (REG_CNT + 2) + +/* A comparison's src2 says whether its flags feed the next branch (bit 0) and, + * above that, the immediate replacing src1 plus A64_CMP_BIAS, or zero for the + * register form. The bias keeps the field positive for a negative immediate. + * A64_CMP_TESTED, past the immediate, says that the AND before it already set + * the flags as ANDS, so that the comparison emits nothing of its own. + */ +#define A64_CMP_FUSED 1 +#define A64_CMP_BIAS 4096 +#define A64_CMP_TESTED (1 << 14) + +/* Added to the src2 of an ADD or SUB immediate that takes LSL #12. */ +#define A64_IMM_SHIFTED 4096 + +/* The option bit that turns a UXTW index of LDR/STR (register) into SXTW. */ +#define A64_SXTW_INDEX 0x8000 + +static int a64_save_bytes(int registers) +{ + return 16 + ALIGN_UP(((registers & 15) + !!(registers & A64_SAVE_GP)) * 8, + A64_STACK_ALIGN); +} + +/* The packed save list ends with x19 only when main replaces its ABI value. */ +static int a64_saved_reg(int saved, int index) +{ + if (index < (saved & 15)) + return 20 + index; + return index == (saved & 15) && (saved & A64_SAVE_GP) ? A64_GP : A64_ZR; +} + +void emit(int insn); +void emit_mop(vir_mop_t *ir); +void a64_mov_imm(int d, int v); +void a64_mov_sp(int d); +void a64_mem(int access, int size, int rt, int rn, int ofs); +void a64_extend(int d, int n, int size, bool is_unsigned); +void a64_bl_addr(int target); + +static void arm64_halfword_counts(unsigned long long value, + int *zero_words, + int *one_words) +{ + *zero_words = 0; + *one_words = 0; + for (int shift = 0; shift < 64; shift += 16) { + unsigned int word = (unsigned int) (value >> shift) & 0xffffU; + *zero_words += word != 0; + *one_words += word != 0xffffU; + } +} + +static void emit_arm64_wide_const(int reg, unsigned long long value) +{ + int zero_words, one_words, first = -1; + bool use_movn; + + /* Writing a W register clears the upper half, so a value confined to the + * low word with one halfword all ones is a single MOVN on it. + */ + if (!(value >> 32) && + ((value & 0xffff) == 0xffff || value >> 16 == 0xffff)) { + int hw = (value & 0xffff) == 0xffff; + emit(a64_movn_insn(false, reg, ~(value >> (16 * hw)) & 0xffff, hw)); + return; + } + arm64_halfword_counts(value, &zero_words, &one_words); + use_movn = one_words < zero_words; + for (int shift = 0; shift < 64; shift += 16) { + unsigned int word = (unsigned int) (value >> shift) & 0xffffU; + bool differs = use_movn ? word != 0xffffU : word != 0; + + if (!differs) + continue; + if (first < 0) { + first = shift / 16; + emit(use_movn ? a64_movn_insn(true, reg, (~word) & 0xffffU, first) + : a64_movz_insn(true, reg, word, first)); + } else { + emit(a64_movk_insn(true, reg, word, shift / 16)); + } + } + if (first < 0) + emit(use_movn ? a64_movn_insn(true, reg, 0, 0) + : a64_movz_insn(true, reg, 0, 0)); +} int a64_reg(int r) { - return r < 8 ? r : r + 12; + if (r == REG_CNT || r == REG_CNT + 1) + return A64_IP0 + r - REG_CNT; + if (r == A64_ZERO) + return A64_ZR; + return r < 8 ? r : r < A64_FIRST_CALLEE ? r + 1 : r + 5; } void emit(int insn) { - elf_write_int(elf_code, insn); + if (mop_count_only) { + if (mop_counted_words < INT_MAX) + mop_counted_words++; + } else { + elf_write_int(elf_code, insn); + } } void a64_mov(int d, int n) { - emit(a64_mov_insn(true, d, n)); + if (d != n) + emit(a64_mov_insn(true, d, n)); } /* SP is not a general register in logical instructions: ORR would read ZR. ADD @@ -44,6 +175,9 @@ void a64_mov_imm(int d, int v) * compiler. A negative C int must therefore become a sign-extended * X-register value, not 0x00000000ffffffff-style zero extension. MOVN * supplies the upper one bits while MOVK fills the second halfword. + * + * Always two instructions: callers pass segment addresses that are not + * final when code size is estimated, so the length cannot depend on v. */ if (v < 0) emit(a64_movn_insn(true, d, ~v, 0)); @@ -52,34 +186,32 @@ void a64_mov_imm(int d, int v) emit(a64_movk_insn(true, d, v >> 16, 1)); } -/* A 32-bit unsigned constant has zeroes above bit 31 when it later widens to an - * X-register scalar. The signed helper intentionally uses MOVN for a negative - * int, so keep this spelling separate. +/* A machine-operation constant is final before sizes are estimated, so unlike + * an address it takes only the halfwords it needs; update_elf_offset() counts + * this very emission. An eight-byte constant keeps its upper word in src1, and + * a smaller one widens to the X register by its C signedness. */ -void a64_mov_imm_unsigned(int d, int v) +static unsigned long long a64_constant_value(const vir_mop_t *p) { - unsigned int value = v; + unsigned long long value; - emit(a64_movz_insn(true, d, value, 0)); - emit(a64_movk_insn(true, d, value >> 16, 1)); + if (p->size_bytes == 8) + value = (unsigned int) p->src0 | + (unsigned long long) (unsigned int) p->src1 << 32; + else if (p->is_unsigned) + value = (unsigned int) p->src0; + else + value = (unsigned long long) (long long) p->src0; + return value; } -/* Phase-2 constants retain their upper word in src1. Materialise all four - * halfwords for an eight-byte scalar rather than silently discarding it in the - * ordinary int helper above. - */ -void a64_mov_imm_wide(int d, int lo, int hi) +static void a64_load_constant(int d, const vir_mop_t *p) { - unsigned int low = lo, high = hi; - - emit(a64_movz_insn(true, d, low, 0)); - emit(a64_movk_insn(true, d, low >> 16, 1)); - emit(a64_movk_insn(true, d, high, 2)); - emit(a64_movk_insn(true, d, high >> 16, 3)); + emit_arm64_wide_const(d, a64_constant_value(p)); } /* Rd = Rn / Rm, unsigned when either operand is. */ -int a64_div_insn(ph2_ir_t *p, int d, int n, int m) +int a64_div_insn(vir_mop_t *p, int d, int n, int m) { bool sf = p->size_bytes == 8; @@ -88,16 +220,48 @@ int a64_div_insn(ph2_ir_t *p, int d, int n, int m) return a64_sdiv_insn(sf, d, n, m); } -/* Rd = Rn >> Rm, logical for an unsigned left operand. */ -int a64_rshift_insn(ph2_ir_t *p, int d, int n, int m) +/* Rd = Rn >> Rm, or by the immediate src2 - 1 when src2 is set; logical for an + * unsigned left operand. + */ +int a64_rshift_insn(vir_mop_t *p, int d, int n, int m) { bool sf = p->size_bytes == 8; + if (p->src2) + return a64_shr_imm_insn(sf, p->src0_is_unsigned, d, n, p->src2 - 1); if (p->src0_is_unsigned) return a64_lsrv_insn(sf, d, n, m); return a64_asrv_insn(sf, d, n, m); } +/* The shift fields of a register operand that a64_fold_shift() shifted, from + * the negative src2 of @p; zero for a plain register. + */ +static int a64_shifted(vir_mop_t *p) +{ + int code = p->src2 < 0 ? -1 - p->src2 : 0; + return ((code & 63) << 10) | ((code >> 6) << 22); +} + +/* The twelve immediate bits of an addition or subtraction with a positive src2, + * which arm64_fold_immediates() sets to the immediate plus one, or for one + * shifted by twelve bits, to A64_IMM_SHIFTED plus the shifted value plus one. + */ +static int a64_imm12(vir_mop_t *p) +{ + return (p->src2 - 1) % A64_IMM_SHIFTED; +} + +/* Whether an addition or subtraction takes the X form. VIR arithmetic carries + * no signedness, so the only int one marked unsigned is one whose result is + * only zero-extended: a64_coalesce_results() gives it the W form, which leaves + * it zero-extended already. + */ +static bool a64_arith_wide(vir_mop_t *p) +{ + return p->size_bytes != 4 || !p->is_unsigned; +} + /* SP cannot be an operand of the shifted-register form, so an addition or * subtraction involving it takes the extended-register form instead. */ @@ -115,22 +279,50 @@ void a64_sub(int d, int n, int m) else emit(a64_sub_reg_insn(true, d, n, m)); } -void a64_sxtw(int d, int n) + +static void a64_adjust_frame(int bytes, bool subtract) { - emit(a64_sext_insn(d, n, 32)); + if (!bytes) + return; + if (bytes < 1 << 24) { + /* The upper twelve bits through the shifted form, then the rest. */ + if (bytes >> 12) + emit((subtract + ? a64_sub_imm_insn(true, A64_SP, A64_SP, bytes >> 12) + : a64_add_imm_insn(true, A64_SP, A64_SP, bytes >> 12)) | + A64_IMM_LSL12); + if (bytes & 0xfff) + emit(subtract + ? a64_sub_imm_insn(true, A64_SP, A64_SP, bytes & 0xfff) + : a64_add_imm_insn(true, A64_SP, A64_SP, bytes & 0xfff)); + } else { + a64_mov_imm(A64_IP0, bytes); + if (subtract) + a64_sub(A64_SP, A64_SP, A64_IP0); + else + a64_add(A64_SP, A64_SP, A64_IP0); + } } - -/* Xd = Xn +/- sign_extend(Wm). The extended-register forms fold the widening of - * an int index into the address arithmetic that consumes it, so pointer - * arithmetic costs the same one instruction as any other add. - */ -void a64_add_sxtw(int d, int n, int m) +/* Rd = SP + @ofs for a non-negative @ofs. */ +static void a64_add_sp(int d, int ofs) { - emit(a64_add_ext_insn(d, n, m, A64_EXT_SXTW)); + if (ofs < 4096) { + emit(a64_add_imm_insn(true, d, A64_SP, ofs)); + return; + } + if (ofs < 1 << 24) { + emit(a64_add_imm_insn(true, d, A64_SP, ofs >> 12) | A64_IMM_LSL12); + if (ofs & 0xfff) + emit(a64_add_imm_insn(true, d, d, ofs & 0xfff)); + return; + } + a64_mov_imm(A64_IP0, ofs); + a64_add(d, A64_SP, A64_IP0); } -void a64_sub_sxtw(int d, int n, int m) + +void a64_sxtw(int d, int n) { - emit(a64_sub_ext_insn(d, n, m, A64_EXT_SXTW)); + emit(a64_sext_insn(d, n, 32)); } /* Sign-extend the low @size bytes of Xn into Xd. Always one instruction, so @@ -151,218 +343,1794 @@ void a64_extend(int d, int n, int size, bool is_unsigned) a64_mov(d, n); } -/* Which load reads @p's value. A narrow unsigned one has to arrive - * zero-extended: its register feeds 64-bit address arithmetic as it stands, and - * there a sign-extended unsigned char of 200 indexes table[-56]. +/* Which load reads @p's value. A narrow unsigned one has to arrive + * zero-extended: its register feeds 64-bit address arithmetic as it stands, and + * there a sign-extended unsigned char of 200 indexes table[-56]. + */ +int a64_load_access(vir_mop_t *p) +{ + return p->is_unsigned ? A64_LOAD_ZEXT : A64_LOAD; +} + +/* AArch64's ordinary LDR/STR immediate is scaled by the access width. The + * compiler deliberately supports packed C layouts, so structure members are + * often not naturally aligned (func_t.bbs, for example). Such offsets must use + * the byte-addressed LDUR/STUR form; rounding them down corrupts adjacent + * fields during self-hosting. + */ +static bool a64_mem_reaches(int size, int ofs) +{ + return (ofs >= -256 && ofs <= 255) || + (ofs >= 0 && ofs <= 4095 * size && !(ofs % size)); +} + +/* Every offset is final when sizes are estimated, so the sequence may depend on + * it. One out of reach of both immediate forms goes through IP0: its upper bits + * added as a shifted immediate when the rest is then in reach, and otherwise + * added whole. + */ +void a64_mem(int access, int size, int rt, int rn, int ofs) +{ + if (size != 1 && size != 2 && size != 4 && size != 8) + fatal("unsupported arm64 access width"); + if (!a64_mem_reaches(size, ofs)) { + if (ofs > 0 && ofs < 1 << 24 && a64_mem_reaches(size, ofs & 0xfff)) { + emit(a64_add_imm_insn(true, A64_IP0, rn, ofs >> 12) | + A64_IMM_LSL12); + ofs &= 0xfff; + } else { + emit_arm64_wide_const(A64_IP0, (long long) ofs); + a64_add(A64_IP0, rn, A64_IP0); + ofs = 0; + } + rn = A64_IP0; + } + if (ofs >= -256 && ofs <= 255 && (ofs < 0 || ofs % size)) + emit(a64_mem_unscaled_insn(access, size, rt, rn, ofs)); + else + emit(a64_mem_scaled_insn(access, size, rt, rn, ofs)); +} + +/* Masking an out-of-range displacement silently branches somewhere else. The + * generated image is small enough that these never fire today, so say so rather + * than let a larger input fail as a wild jump. + */ +void a64_check_disp(int disp, int bits, char *what) +{ + if (mop_count_only) + return; + int lim = 1 << (bits - 1); + if (disp < -lim || disp >= lim) + fatal(what); +} + +/* Branch to @target when @rt is non-zero. CBNZ carries the same 19-bit + * displacement a B.cond would, so it replaces a compare-and-branch pair + * outright. @wide picks the X form, which a pointer needs: testing only Wn + * reads an address whose low word happens to be zero as null. + */ +void a64_cbnz(bool wide, int rt, int target) +{ + int disp = (target - elf_code->size) / 4; + a64_check_disp(disp, 19, "arm64 conditional branch out of range"); + emit(a64_cbnz_insn(wide, rt, disp)); +} +void a64_b(int target) +{ + int d = (target - elf_code->size) / 4; + a64_check_disp(d, 26, "arm64 branch out of range"); + emit(a64_b_insn(d)); +} +void a64_bl_addr(int target) +{ + int d = (target - (elf_code_start + elf_code->size)) / 4; + a64_check_disp(d, 26, "arm64 call out of range"); + emit(a64_bl_insn(d)); +} +int a64_adrp_insn(int d, int pc, int target) +{ + int pages = (target >> 12) - (pc >> 12); + a64_check_disp(pages, 21, "arm64 ADRP target out of range"); + return a64_adrp_pages_insn(d, pages); +} + +/* A comparison is as wide as the values it compares. An address has to be + * compared whole, an array included, since what reaches the instruction is its + * decayed base. The source flags say so and is_pointer does not: it counts + * pointer-like operands alone, for the sake of the 32-bit targets that read it. + */ +bool a64_cmp_wide(vir_mop_t *p) +{ + return p->size_bytes == 8 || p->src0_is_pointer || p->src1_is_pointer; +} + +a64_cond_t a64_cond(opcode_t op, bool is_unsigned) +{ + switch (op) { + case OP_eq: + return A64_EQ; + case OP_neq: + return A64_NE; + case OP_geq: + return is_unsigned ? A64_HS : A64_GE; + case OP_lt: + return is_unsigned ? A64_LO : A64_LT; + case OP_gt: + return is_unsigned ? A64_HI : A64_GT; + default: + return is_unsigned ? A64_LS : A64_LE; + } +} + +/* Whether control leaving @bb at its end lands in @target: the next block in + * layout, or one behind blocks that emit nothing, such as the arms a select + * diamond emptied. + */ +static bool a64_falls_into(basic_block_t *bb, basic_block_t *target) +{ + for (basic_block_t *next = bb->rpo_next; next; next = next->rpo_next) { + if (next == target) + return true; + if (next->mops.head) + return false; + } + return false; +} + +/* Which successor of a branch, if any, control falls into after @bb. */ +static int a64_branch_fallthrough(basic_block_t *bb, vir_mop_t *ir) +{ + return a64_falls_into(bb, ir->else_bb) ? 1 + : a64_falls_into(bb, ir->then_bb) ? 2 + : 0; +} + +static bool a64_branch_only_compare(basic_block_t *bb, vir_mop_t *ir) +{ + return op_is_comparison(ir->op) && ir->next && ir->next->op == OP_branch && + ir->next->src0 == ir->dest && + !(bb->machine_liveout & (1U << ir->dest)); +} + +/* Turn the branch after an equality test against zero into CBZ or CBNZ on the + * tested register, which makes @compare redundant. The branch reads its operand + * at the comparison's width, and an equality test swaps the successors, since + * the branch is taken on a nonzero operand. + */ +static bool a64_branch_on_zero(vir_mop_t *compare, vir_mop_t *branch) +{ + if ((compare->op != OP_eq && compare->op != OP_neq) || + compare->src2 >> 1 != A64_CMP_BIAS) + return false; + branch->src0 = compare->src0; + branch->size_bytes = a64_cmp_wide(compare) ? 8 : 4; + branch->src0_is_pointer = false; + if (compare->op == OP_eq) { + basic_block_t *then_bb = branch->then_bb; + branch->then_bb = branch->else_bb; + branch->else_bb = then_bb; + } + return true; +} + +/* Count the actual instruction sequence so offset planning stays in step with + * the emitter as it changes. + */ +void update_elf_offset(vir_mop_t *ir) +{ + if (ir->op != OP_allocat) + elf_offset += mop_count_words(ir) * 4; +} + +/* Registers @f may name anywhere, over-approximated by every operand field. */ +static unsigned int a64_registers_named(func_t *f) +{ + unsigned int named = 0; + for (basic_block_t *bb = f->bbs; bb; bb = bb->rpo_next) + for (vir_mop_t *ir = bb->mops.head; ir; ir = ir->next) { + int fields[] = {ir->dest, ir->src0, ir->src1, + ir->dest_hi, ir->src0_hi, ir->src1_hi}; + for (int i = 0; i < 6; i++) + if (fields[i] >= 0 && fields[i] < REG_CNT) + named |= 1u << fields[i]; + } + return named; +} + +/* The registers a64_hoist_constants() gave the current function's literals, and + * the literal each holds from its entry on. + */ +static unsigned int a64_lanes; +static unsigned long long a64_lane_values[REG_CNT]; + +/* Loop literals use otherwise idle callee-saved registers. Their definitions + * move to function entry, unless a successor or call needs the literal in its + * own register. Values and addresses outside this literal-only pass are left to + * the shared lowerer. + */ +static void a64_hoist_constants(func_t *f) +{ + int count = 0, lanes = 0; + vir_mop_t *constants[CALLEE_SAVED_REGS]; + int lane[CALLEE_SAVED_REGS]; + + a64_lanes = 0; + + /* A leaf keeps its literals in caller-saved registers it never names, which + * need no save; anything else uses idle callee-saved ones. + */ + if (!func_makes_call(f)) { + unsigned int named = a64_registers_named(f); + for (int reg = A64_FIRST_CALLEE - 1; + reg > 0 && lanes < CALLEE_SAVED_REGS; reg--) + if (!(named & (1u << reg))) + lane[lanes++] = reg; + } else + for (int reg = func_highest_used_reg(f, A64_FIRST_CALLEE) + 1; + reg < REG_CNT; reg++) + lane[lanes++] = reg; + if (!lanes) + return; + + /* Literals leave loop bodies only: the blocks that reach a back edge + * without passing the header it returns to. A literal after a loop's exit, + * or between two loops, would hold a lane across the loop for nothing. + * Blocks keep their order along rpo_next, so one backward sweep from each + * back edge finds its body. + */ + int blocks = 0; + for (basic_block_t *bb = f->bbs; bb; bb = bb->rpo_next) + if (bb->rpo >= blocks) + blocks = bb->rpo + 1; + basic_block_t **order = calloc(blocks, sizeof(*order)); + char *body = calloc(blocks, 1), *in_loop = calloc(blocks, 1); + if (!order || !body || !in_loop) + fatal("Out of memory"); + for (basic_block_t *bb = f->bbs; bb; bb = bb->rpo_next) + order[bb->rpo] = bb; + for (basic_block_t *latch = f->bbs; latch; latch = latch->rpo_next) { + basic_block_t *exits[3] = {latch->next, latch->then_, latch->else_}; + for (int e = 0; e < 3; e++) { + basic_block_t *header = exits[e]; + if (!header || header->rpo > latch->rpo) + continue; + for (int i = 0; i < blocks; i++) + body[i] = 0; + body[latch->rpo] = 1; + for (int i = latch->rpo - 1; i >= header->rpo; i--) { + basic_block_t *bb = order[i]; + if (!bb) + continue; + basic_block_t *targets[3] = {bb->next, bb->then_, bb->else_}; + for (int t = 0; t < 3; t++) + if (targets[t] && targets[t] != header && + targets[t]->rpo > i && targets[t]->rpo <= latch->rpo && + body[targets[t]->rpo]) + body[i] = 1; + } + body[header->rpo] = 1; + for (int i = header->rpo; i <= latch->rpo; i++) + in_loop[i] |= body[i]; + } + } + for (basic_block_t *bb = f->bbs; bb; bb = bb->rpo_next) { + if (!in_loop[bb->rpo]) + continue; + vir_mop_t **link = &bb->mops.head, *previous = NULL; + while (*link) { + vir_mop_t *ir = *link; + if (ir->op != OP_load_constant) { + previous = ir; + link = &ir->next; + continue; + } + + /* A literal that has to stay in its own register, for a successor + * or across a call, would only become a copy of its lane, which is + * no faster and costs the lane. + */ + int old = ir->dest; + bool copy = (bb->machine_liveout & (1u << old)) != 0; + for (vir_mop_t *use = ir->next; !copy && use; use = use->next) { + if (use->op == OP_call || use->op == OP_indirect) { + copy = true; + break; + } + if (op_writes_dest(use->op) && use->dest == old) + break; + } + unsigned long long value = a64_constant_value(ir); + int index = 0; + while (index < count && + a64_constant_value(constants[index]) != value) + index++; + if (copy || (index == count && count >= lanes)) { + previous = ir; + link = &ir->next; + continue; + } + if (index == count) { + vir_mop_t *setup = mop_create(OP_load_constant); + *setup = *ir; + setup->dest = lane[count]; + setup->next = NULL; + if (count) + constants[count - 1]->next = setup; + constants[count++] = setup; + } + int reg = constants[index]->dest; + for (vir_mop_t *use = ir->next; use; use = use->next) { + if (op_src0_is_reg(use->op) && use->src0 == old) + use->src0 = reg; + if (op_src1_is_reg(use->op) && use->src1 == old) + use->src1 = reg; + if (use->src0_hi == old) + use->src0_hi = reg; + if (op_writes_dest(use->op) && use->dest == old) + break; + } + *link = ir->next; + if (bb->mops.tail == ir) + bb->mops.tail = previous; + } + } + free(order); + free(body); + free(in_loop); + for (int i = 0; i < count; i++) { + a64_lanes |= 1u << constants[i]->dest; + a64_lane_values[constants[i]->dest] = a64_constant_value(constants[i]); + } + if (count) { + constants[count - 1]->next = f->bbs->mops.head; + f->bbs->mops.head = constants[0]; + if (!f->bbs->mops.tail) + f->bbs->mops.tail = constants[count - 1]; + } +} + +/* Whether what @def wrote is dead once @use, the next operation, has run. */ +static bool a64_value_dies(basic_block_t *bb, vir_mop_t *def, vir_mop_t *use) +{ + return (op_writes_dest(use->op) && use->dest == def->dest) || + !reg_read_after(bb, use, def->dest); +} + +/* Turn an AND that only an equality test against zero reads into TST, the ANDS + * that keeps no result, for a comparison that sets a register or selects one. + * Before a branch, the test becomes CBZ or CBNZ, which is no longer. + */ +static bool a64_test_bits(basic_block_t *bb, vir_mop_t *and) +{ + vir_mop_t *compare = and->next; + if (and->op != OP_bit_and || !compare || + (compare->op != OP_eq && compare->op != OP_neq) || + compare->src0 != and->dest || compare->src2 >> 1 != A64_CMP_BIAS || + (compare->next && compare->next->op == OP_branch) || + a64_cmp_wide(compare) != (and->size_bytes == 8) || + !a64_value_dies(bb, and, compare)) + return false; + and->dest = A64_ZERO; + compare->src2 |= A64_CMP_TESTED; + return true; +} + +/* Whether @extend widens exactly what the load @load read, so that the load can + * extend it itself: LDRSB, LDRSH and LDRSW sign-extend into the X register, and + * the plain narrow loads zero-extend. + */ +static bool a64_fold_load_extend(vir_mop_t *load, vir_mop_t *extend) +{ + int size = load->op == OP_read ? load->src1 : load->size_bytes; + int dst_size = extend->src1 & 0xffff; + + return (load->op == OP_load || load->op == OP_global_load || + load->op == OP_read) && + extend->op == OP_sign_ext && extend->src0 == load->dest && + (extend->src1 >> 16) == size && size < 8 && + !(dst_size == PTR_SIZE && extend->is_pointer); +} + +/* Rewrite @use so that it no longer reads the dead literal @value from @reg, + * where AArch64 has a form for it: a comparison against an immediate, a store + * of the zero register, or an extension evaluated now. + */ +static bool a64_fold_literal(int reg, vir_mop_t *use, long long value) +{ + if (op_is_comparison(use->op) && (use->src0 == reg) != (use->src1 == reg) && + value >= -4095 && value <= 4095) { + /* With the literal on the left, swap the operands and mirror the + * comparison; equality reads the same either way. + */ + if (use->src0 == reg) { + use->src0 = use->src1; + if (use->op == OP_lt) + use->op = OP_gt; + else if (use->op == OP_gt) + use->op = OP_lt; + else if (use->op == OP_leq) + use->op = OP_geq; + else if (use->op == OP_geq) + use->op = OP_leq; + } + use->src1 = -1; + use->src2 = ((int) value + A64_CMP_BIAS) << 1; + return true; + } + if (!value && (use->op == OP_store || use->op == OP_global_store) && + use->src0 == reg) { + use->src0 = A64_ZERO; + return true; + } + if (!value && use->op == OP_write && use->src1 == reg && use->src0 != reg && + use->src0_hi != reg) { + use->src1 = A64_ZERO; + return true; + } + + /* x * (2^k + 1) is x + (x << k). Only the low word of an int product is its + * value, so the X form serves either width. + */ + if (use->op == OP_mul && (use->src0 == reg) != (use->src1 == reg) && + value > 2 && value <= (1LL << (use->size_bytes == 8 ? 32 : 31)) + 1 && + !((value - 1) & (value - 2))) { + int shift = 0; + while ((1LL << shift) + 1 != value) + shift++; + use->op = OP_add; + if (use->src0 == reg) + use->src0 = use->src1; + use->src1 = use->src0; + use->src2 = -shift - 1; + return true; + } + if ((use->op == OP_trunc || use->op == OP_sign_ext) && use->src0 == reg) { + /* Decode the width exactly as emit_mop() does for these two. */ + int size = use->src1; + bool is_unsigned = use->is_unsigned; + if (use->op == OP_sign_ext) { + size = (use->src1 >> 16) & 0xffff; + is_unsigned = use->src0_is_unsigned; + if ((use->src1 & 0xffff) == PTR_SIZE && use->is_pointer) + size = 8; + } + unsigned long long bits = value; + if (size == 1 || size == 2 || size == 4) { + unsigned long long sign = 1ULL << (size * 8 - 1); + bits &= (sign << 1) - 1; + if (!is_unsigned && (bits & sign)) + bits -= sign << 1; + } + use->op = OP_load_constant; + use->size_bytes = 8; + use->src0 = (int) bits; + use->src1 = (int) (bits >> 32); + return true; + } + return false; +} + +/* Whether @x keeps @def from moving below it: it reads or writes what @def + * writes, overwrites what @def reads, or ends the block or calls out. + */ +static bool a64_blocks_sink(vir_mop_t *def, vir_mop_t *x) +{ + if (x->op == OP_branch || x->op == OP_jump || x->op == OP_return || + x->op == OP_call || x->op == OP_indirect || + ir_reads_reg(x, def->dest) || (mop_defs(x) & (1u << def->dest))) + return true; + unsigned int clobbers = mop_defs(x); + for (int reg = 0; reg < REG_CNT; reg++) + if ((clobbers & (1u << reg)) && ir_reads_reg(def, reg)) + return true; + return false; +} + +/* What a64_sink_defs() moves: literals, then the address arithmetic of a memory + * access, then the extension of its index, and last a product to its addition. + */ +enum { A64_SINK_LITERAL, A64_SINK_ADDRESS, A64_SINK_INDEX, A64_SINK_PRODUCT }; + +static bool a64_sinks(int kind, vir_mop_t *def, vir_mop_t *reader) +{ + bool access = reader->op == OP_read || reader->op == OP_write; + if (kind == A64_SINK_LITERAL) + return def->op == OP_load_constant; + if (kind == A64_SINK_ADDRESS) + return (((def->op == OP_add || def->op == OP_sub) && + def->size_bytes == 8) || + def->op == OP_address_of || def->op == OP_global_address_of) && + access && reader->src0 == def->dest; + if (kind == A64_SINK_PRODUCT) + return def->op == OP_mul && reader->op == OP_add; + return def->op == OP_sign_ext && access && reader->src0_hi == def->dest; +} + +/* Move each operation of @kind down to the first one that reads its result, so + * that the folds, which pair an operation with the one after it, find it next + * to its use. Later ones move first, so that several of them end up in their + * order in front of a reader. + */ +static void a64_sink_defs(func_t *f, int kind) +{ + int capacity = 0; + vir_mop_t **order = NULL; + for (basic_block_t *bb = f->bbs; bb; bb = bb->rpo_next) { + int count = 0; + for (vir_mop_t *ir = bb->mops.head; ir; ir = ir->next) + count++; + if (count < 3) + continue; + if (count > capacity) { + free(order); + capacity = count * 2; + order = malloc(capacity * sizeof(*order)); + if (!order) + fatal("Out of memory"); + } + count = 0; + for (vir_mop_t *ir = bb->mops.head; ir; ir = ir->next) + order[count++] = ir; + for (int i = count - 2; i >= 0; i--) { + vir_mop_t *def = order[i]; + if (!op_writes_dest(def->op) || def->dest_hi >= 0 || + def->dest < 0 || def->dest >= REG_CNT) + continue; + int at = i + 1; + while (at < count && !a64_blocks_sink(def, order[at])) + at++; + if (at == i + 1 || at == count || + !ir_reads_reg(order[at], def->dest) || + !a64_sinks(kind, def, order[at])) + continue; + for (int k = i; k < at - 1; k++) + order[k] = order[k + 1]; + order[at - 1] = def; + } + bb->mops.head = order[0]; + for (int i = 0; i < count - 1; i++) + order[i]->next = order[i + 1]; + order[count - 1]->next = NULL; + bb->mops.tail = order[count - 1]; + } + free(order); +} + +/* Feed the shift by an immediate @shift, whose result only @use reads, to @use + * as its shifted second operand: ADD and SUB on X registers, and AND, ORR and + * EOR on registers as wide as the shift's. src2 of @use then encodes the shift + * as -1 - (amount + 64 * kind), kind being 0 for LSL, 1 for LSR and 2 for ASR. + */ +static bool a64_fold_shift(basic_block_t *bb, vir_mop_t *shift, vir_mop_t *use) +{ + int r = shift->dest; + bool logic = + use->op == OP_bit_and || use->op == OP_bit_or || use->op == OP_bit_xor; + if ((shift->op != OP_lshift && shift->op != OP_rshift) || + shift->src2 <= 0 || use->src2 || (use->src0 == r) == (use->src1 == r) || + (use->src0 == r && use->op == OP_sub) || + !(logic ? shift->size_bytes == use->size_bytes + : (use->op == OP_add || use->op == OP_sub) && + shift->size_bytes == 8 && use->size_bytes == 8) || + !a64_value_dies(bb, shift, use)) + return false; + int kind = shift->op == OP_lshift ? 0 : shift->src0_is_unsigned ? 1 : 2; + if (use->src0 == r) + use->src0 = use->src1; + use->src1 = shift->src0; + use->src2 = -1 - (shift->src2 - 1 + 64 * kind); + return true; +} + +/* Fold a dead adjacent literal into the arithmetic instruction. src2 is an + * opcode-local immediate plus one; zero retains the register form. + */ +static void arm64_fold_immediates(func_t *func) +{ + a64_sink_defs(func, A64_SINK_LITERAL); + for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { + vir_mop_t **link = &bb->mops.head; + while (*link && (*link)->next) { + vir_mop_t *literal = *link, *use = literal->next; + long long value = (long long) a64_constant_value(literal); + if (literal->op == OP_load_constant && + a64_value_dies(bb, literal, use) && + a64_fold_literal(literal->dest, use, value)) { + *link = use; + continue; + } + if (a64_fold_load_extend(literal, use) && + a64_value_dies(bb, literal, use)) { + literal->is_unsigned = use->src0_is_unsigned; + literal->dest = use->dest; + literal->next = use->next; + if (bb->mops.tail == use) + bb->mops.tail = literal; + continue; + } + if (literal->op == OP_load_constant && !use->src2 && + (use->src0 == literal->dest) != (use->src1 == literal->dest) && + (use->op == OP_bit_and || use->op == OP_bit_or || + use->op == OP_bit_xor) && + a64_value_dies(bb, literal, use)) { + int field = a64_bitmask_field(a64_constant_value(literal), + use->size_bytes == 8); + if (field >= 0) { + if (use->src0 == literal->dest) + use->src0 = use->src1; + use->src2 = field + 1; + use->src1 = -1; + *link = use; + continue; + } + } + if (a64_fold_shift(bb, literal, use)) { + *link = use; + continue; + } + /* An addition takes the literal from either side. */ + if (literal->op == OP_load_constant && use->op == OP_add && + !use->src2 && use->src0 == literal->dest && + use->src1 != literal->dest) { + use->src0 = use->src1; + use->src1 = literal->dest; + } + /* Unsigned, so that LLONG_MIN stays out of range too. */ + unsigned long long magnitude = + value < 0 ? 0ULL - (unsigned long long) value + : (unsigned long long) value; + if (literal->op != OP_load_constant || use->src1 != literal->dest || + use->src0 == literal->dest || use->src2 || + (use->op != OP_add && use->op != OP_sub && + use->op != OP_lshift && use->op != OP_rshift) || + !a64_value_dies(bb, literal, use) || + (use->op == OP_lshift || use->op == OP_rshift + ? value < 0 || value >= (use->size_bytes == 8 ? 64 : 32) + : magnitude > 4095 && + (magnitude & 0xfff || magnitude >= 1 << 24))) { + link = &literal->next; + continue; + } + if (value < 0) { + use->op = use->op == OP_add ? OP_sub : OP_add; + value = -value; + } + + /* Past twelve bits, the immediate is a multiple of 4096 that the + * instruction shifts: src2 then exceeds 4096 by that multiple. + */ + use->src2 = (value > 4095 ? A64_IMM_SHIFTED + (int) (value >> 12) + : (int) value) + + 1; + use->src1 = -1; + *link = use; + } + } + a64_sink_defs(func, A64_SINK_ADDRESS); + for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { + vir_mop_t **link = &bb->mops.head; + while (*link && (*link)->next) { + vir_mop_t *address = *link, *memory = address->next; + int size = memory->op == OP_read ? memory->src1 : memory->dest; + if (address->size_bytes != 8 || address->src2 > A64_IMM_SHIFTED || + (address->op != OP_add && address->op != OP_sub) || + (memory->op != OP_read && memory->op != OP_write) || + (address->src2 <= 0 && + (address->op != OP_add || address->src2 < -4 || + (address->src2 < -1 && + (1 << (-address->src2 - 1)) != size))) || + memory->src0_hi >= 0 || memory->src2 != 0 || + memory->src0 != address->dest || + (memory->op == OP_write && memory->src1 == address->dest) || + !a64_value_dies(bb, address, memory)) { + link = &address->next; + continue; + } + memory->src0 = address->src0; + memory->src0_hi = address->src2 <= 0 ? address->src1 : -1; + memory->src2 = + address->src2 <= 0 + ? (address->src2 ? address->src2 : -1) + : (address->src2 - 1) * (address->op == OP_sub ? -1 : 1); + *link = memory; + } + } + + /* An access through the address of a frame slot or a global becomes an + * access to that slot or global, which takes its offset as an immediate. + * The address may have reached the access only through the fold above. + */ + a64_sink_defs(func, A64_SINK_ADDRESS); + for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { + vir_mop_t **link = &bb->mops.head; + while (*link && (*link)->next) { + vir_mop_t *address = *link, *memory = address->next; + bool read = memory->op == OP_read; + int offset = address->src0 + memory->src2; + if ((address->op != OP_address_of && + address->op != OP_global_address_of) || + (!read && memory->op != OP_write) || memory->src0_hi >= 0 || + memory->src0 != address->dest || offset < 0 || + (!read && memory->src1 == address->dest) || + !a64_value_dies(bb, address, memory)) { + link = &address->next; + continue; + } + bool global = address->op == OP_global_address_of; + if (read) { + memory->op = global ? OP_global_load : OP_load; + memory->size_bytes = memory->src1; + memory->src0 = offset; + memory->src1 = -1; + } else { + memory->op = global ? OP_global_store : OP_store; + memory->size_bytes = memory->dest; + memory->src0 = memory->src1; + memory->src1 = offset; + memory->dest = -1; + } + memory->src2 = 0; + memory->ofs_based_on_stack_top = address->ofs_based_on_stack_top; + *link = memory; + } + } + + /* An int index extends in the access itself: src2 from -5 to -8 for UXTW + * and from -9 to -12 for SXTW, each shifting by the access width or not. + */ + a64_sink_defs(func, A64_SINK_INDEX); + for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { + vir_mop_t **link = &bb->mops.head; + while (*link && (*link)->next) { + vir_mop_t *extend = *link, *memory = extend->next; + if (extend->op != OP_sign_ext || extend->src1 != ((4 << 16) | 8) || + extend->is_pointer || + (memory->op != OP_read && memory->op != OP_write) || + memory->src0_hi != extend->dest || + memory->src0 == extend->dest || memory->src2 > -1 || + memory->src2 < -4 || + (memory->op == OP_write && memory->src1 == extend->dest) || + !a64_value_dies(bb, extend, memory)) { + link = &extend->next; + continue; + } + memory->src0_hi = extend->src0; + memory->src2 -= extend->src0_is_unsigned ? 4 : 8; + *link = memory; + } + } +} + +/* A multiplication whose product only an addition of the same width reads + * becomes MADD, which takes the other addend in src1_hi. A W-form addition + * pairs with a 32-bit product; an int one in X form keeps upper bits that a W + * MADD would clear, so it stays apart. + */ +static void a64_fuse_multiply_add(func_t *f) +{ + a64_sink_defs(f, A64_SINK_PRODUCT); + for (basic_block_t *bb = f->bbs; bb; bb = bb->rpo_next) + for (vir_mop_t **link = &bb->mops.head; *link && (*link)->next;) { + vir_mop_t *mul = *link, *add = mul->next; + int r = mul->dest; + if (mul->op != OP_mul || mul->src2 || mul->dest_hi >= 0 || + mul->src1_hi >= 0 || add->op != OP_add || add->src2 || + add->dest_hi >= 0 || add->size_bytes != mul->size_bytes || + (mul->size_bytes != 8 && a64_arith_wide(add)) || + (add->src0 == r) == (add->src1 == r) || + !a64_value_dies(bb, mul, add)) { + link = &mul->next; + continue; + } + add->src1_hi = add->src0 == r ? add->src1 : add->src0; + add->op = OP_mul; + add->src0 = mul->src0; + add->src1 = mul->src1; + *link = add; + } +} + +/* Whether @ir only computes its destination register, with no other effect. */ +static bool a64_pure(vir_mop_t *ir) +{ + switch (ir->op) { + case OP_load_constant: + case OP_address_of: + case OP_global_address_of: + case OP_load_rodata_address: + case OP_assign: + case OP_cast: + case OP_trunc: + case OP_sign_ext: + case OP_add: + case OP_sub: + case OP_mul: + case OP_lshift: + case OP_rshift: + case OP_bit_and: + case OP_bit_or: + case OP_bit_xor: + return ir->dest >= 0 && ir->dest < REG_CNT && ir->dest_hi < 0; + default: + return false; + } +} + +/* Drop pure operations whose result nothing reads. Folding an address or a + * literal into its use leaves the definition it came from in another block, and + * live sets recomputed from what the blocks now read show it unused. + */ +static void a64_drop_dead(func_t *f) +{ + int capacity = 0; + vir_mop_t **order = NULL; + mop_function_liveness(f); + for (basic_block_t *bb = f->bbs; bb; bb = bb->rpo_next) { + int count = 0; + for (vir_mop_t *ir = bb->mops.head; ir; ir = ir->next) + count++; + if (count > capacity) { + free(order); + capacity = count * 2; + order = malloc(capacity * sizeof(*order)); + if (!order) + fatal("Out of memory"); + } + count = 0; + for (vir_mop_t *ir = bb->mops.head; ir; ir = ir->next) + order[count++] = ir; + unsigned int live = bb->machine_liveout; + vir_mop_t *next = NULL; + for (int i = count - 1; i >= 0; i--) { + vir_mop_t *ir = order[i]; + if (a64_pure(ir) && !(live & (1u << ir->dest))) + continue; + live = (live & ~mop_defs(ir)) | mop_reads(ir); + if (!next) + bb->mops.tail = ir; + ir->next = next; + next = ir; + } + bb->mops.head = next; + if (!next) + bb->mops.tail = NULL; + } + free(order); +} + +/* How many branches and jumps in @f name @target. */ +static int a64_branches_naming(func_t *f, basic_block_t *target) +{ + int count = 0; + for (basic_block_t *bb = f->bbs; bb; bb = bb->rpo_next) { + vir_mop_t *tail = bb->mops.tail; + if (tail && tail->op == OP_jump) + count += tail->next_bb == target; + else if (tail && tail->op == OP_branch) + count += (tail->then_bb == target) + (tail->else_bb == target); + } + return count; +} + +/* Join a block that only the jump ending the block before it reaches, an edge + * copy block most often, to that block. The peephole then sees a definition and + * the edge's move of it together, where it can write the result in place. + */ +static void a64_merge_blocks(func_t *f) +{ + for (basic_block_t *bb = f->bbs; bb;) { + vir_mop_t *jump = bb->mops.tail; + basic_block_t *next = + jump && jump->op == OP_jump ? jump->next_bb : NULL; + if (!next || !next->mops.head || !a64_falls_into(bb, next) || + a64_branches_naming(f, next) != 1) { + bb = bb->rpo_next; + continue; + } + vir_mop_t **link = &bb->mops.head; + while (*link != jump) + link = &(*link)->next; + *link = next->mops.head; + bb->mops.tail = next->mops.tail; + next->mops.head = next->mops.tail = NULL; + bb->next = next->next; + bb->then_ = next->then_; + bb->else_ = next->else_; + bb->machine_liveout = next->machine_liveout; + next->next = next->then_ = next->else_ = NULL; + for (vir_mop_t *ir = bb->mops.head; ir; ir = ir->next) + for (int tries = 0; tries < 4 && peephole_at(bb, ir); tries++) + ; + } +} + +/* Whether @use only copies a register, emitted as a plain move. A comparison + * leaves 0 or 1, which every extension of it keeps as it is. + */ +static bool a64_is_copy(vir_mop_t *def, vir_mop_t *use) +{ + if (use->dest_hi >= 0 || use->src0_hi >= 0 || use->src0 != def->dest) + return false; + if (use->op == OP_assign || use->op == OP_cast) + return true; + if (use->op != OP_trunc && use->op != OP_sign_ext) + return false; + if (op_is_comparison(def->op) || def->op == OP_log_not) + return true; + + /* A W-form result is zero-extended already, and an int addition or + * subtraction can take that form. + */ + bool zero_extends = use->op == OP_trunc + ? use->src1 == 4 && use->is_unsigned + : use->src1 == ((4 << 16) | 8) && + use->src0_is_unsigned && !use->is_pointer; + if (!zero_extends || def->size_bytes != 4) + return false; + switch (def->op) { + case OP_add: + case OP_sub: + return def->src2 >= -32; + case OP_mul: + case OP_div: + case OP_mod: + case OP_lshift: + case OP_rshift: + case OP_bit_and: + case OP_bit_or: + case OP_bit_xor: + case OP_negate: + case OP_bit_not: + return true; + default: + return false; + } +} + +/* Write a result where its only reader wants it: the register of a move that + * copies it, or x0 for the return of it. The definition moves down to that + * reader past operations that leave its register and its operands alone, as an + * edge copy is often apart from the constant it copies. The shared peephole + * left these, as it only retargets adjacent instructions fusible everywhere. + */ +static void a64_coalesce_results(func_t *f) +{ + for (basic_block_t *bb = f->bbs; bb; bb = bb->rpo_next) + for (vir_mop_t **link = &bb->mops.head; *link;) { + vir_mop_t *def = *link, *use = def->next; + int r = def->dest; + if (!op_writes_dest(def->op) || def->dest_hi >= 0 || r < 0 || + r >= REG_CNT) { + link = &def->next; + continue; + } + + /* A load moves only into a copy right after it, so that it never + * passes a store or a call. + */ + bool loads = def->op == OP_load || def->op == OP_global_load || + def->op == OP_read; + for (; use; use = use->next) { + if (loads && !a64_is_copy(def, use)) { + use = NULL; + break; + } + if (a64_is_copy(def, use) || ir_reads_reg(use, r) || + (mop_defs(use) & (1u << r)) || use->op == OP_branch || + use->op == OP_jump || use->op == OP_return) + break; + unsigned int clobbers = mop_defs(use); + bool sources = false; + for (int reg = 0; reg < REG_CNT; reg++) + if ((clobbers & (1u << reg)) && ir_reads_reg(def, reg)) + sources = true; + if (sources) + break; + } + int to = -1; + if (use && use->op == OP_return && use->src0 == r && + use->src0_hi < 0 && r) + to = 0; + else if (use && a64_is_copy(def, use) && + a64_value_dies(bb, def, use)) + to = use->dest; + if (to < 0) { + link = &def->next; + continue; + } + + /* The definition takes the place of the copy, or goes ahead of the + * return, which then finds its value in x0. + */ + if (use->op == OP_return) { + vir_mop_t *ret = mop_create(OP_return); + *ret = *use; + ret->src0 = 0; + if (bb->mops.tail == use) + bb->mops.tail = ret; + use->next = ret; + } + vir_mop_t *next = use->next; + bool extends = use->op == OP_trunc || use->op == OP_sign_ext; + *use = *def; + use->dest = to; + use->next = next; + if (extends && (use->op == OP_add || use->op == OP_sub)) + use->is_unsigned = true; + *link = def->next; + } +} + +/* Select between short, effect-free expression arms without a data-dependent + * branch. Only a single live result may change; both scratch evaluations read + * the original inputs and retain the comparison flags. + */ +static void a64_select_diamonds(func_t *func) +{ + if (func_has_op(func, OP_load_func, OP_indirect)) + return; + for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { + vir_mop_t *branch = bb->mops.tail, *compare = NULL; + if (!branch || branch->op != OP_branch) + continue; + for (vir_mop_t *ir = bb->mops.head; ir; ir = ir->next) + if (ir->next == branch) + compare = ir; + if (!compare || !a64_branch_only_compare(bb, compare)) + continue; + basic_block_t *arms[2][4]; + vir_mop_t *ops[2][4]; + int lengths[2] = {0}, nops[2] = {0}, join_at[2] = {-1, -1}; + bool safe = true; + for (int side = 0; side < 2; side++) { + basic_block_t *arm = side ? branch->else_bb : branch->then_bb; + while (arm && lengths[side] < 4) { + arms[side][lengths[side]++] = arm; + vir_mop_t *tail = arm->mops.tail; + arm = tail && tail->op == OP_jump ? tail->next_bb : NULL; + } + } + for (int left = 0; left < lengths[0] && join_at[0] < 0; left++) + for (int right = 0; right < lengths[1]; right++) + if (arms[0][left] == arms[1][right]) { + join_at[0] = left; + join_at[1] = right; + break; + } + if (join_at[0] <= 0 || join_at[1] <= 0) + continue; + int output = -1; + for (int side = 0; side < 2 && safe; side++) { + unsigned int written = 0; + int last = -1; + for (int index = 0; index < join_at[side] && safe; index++) { + basic_block_t *arm = arms[side][index]; + if (!bb_sole_pred(arm)) { + safe = false; + break; + } + for (vir_mop_t *ir = arm->mops.head; ir; ir = ir->next) { + if (ir->op == OP_jump) + continue; + if (nops[side] == 4 || + (ir->op != OP_add && ir->op != OP_sub && + ir->op != OP_bit_xor && ir->op != OP_bit_and && + ir->op != OP_bit_or && ir->op != OP_assign) || + ((written & (1u << ir->src0)) && ir->src0 != last) || + (op_src1_is_reg(ir->op) && ir->src1 >= 0 && + (written & (1u << ir->src1)) && ir->src1 != last)) { + safe = false; + break; + } + ops[side][nops[side]++] = ir; + written |= 1u << ir->dest; + last = ir->dest; + } + } + if (!nops[side]) + safe = false; + else { + unsigned int live = + written & arms[side][join_at[side] - 1]->machine_liveout; + if (!live || (live & (live - 1)) || !(live & (1u << last)) || + (output >= 0 && output != last)) + safe = false; + output = last; + } + } + if (!safe) + continue; + vir_mop_t *tail = compare; + compare->src2 |= A64_CMP_FUSED; + for (int side = 0; side < 2; side++) { + int last = -1, scratch = REG_CNT + side; + for (int index = 0; index < nops[side]; index++) { + vir_mop_t *old = ops[side][index]; + vir_mop_t *ir = mop_create(old->op); + *ir = *old; + ir->next = NULL; + if (ir->src0 == last) + ir->src0 = scratch; + if (op_src1_is_reg(ir->op) && ir->src1 == last) + ir->src1 = scratch; + ir->dest = scratch; + last = old->dest; + tail->next = ir; + tail = ir; + } + for (int index = 0; index < join_at[side]; index++) { + arms[side][index]->mops.head = NULL; + arms[side][index]->mops.tail = NULL; + } + } + vir_mop_t *select = mop_create(OP_ternary); + select->dest = output; + select->src0 = REG_CNT; + select->src1 = REG_CNT + 1; + select->src2 = a64_cond(compare->op, compare->src0_is_unsigned || + compare->src1_is_unsigned); + tail->next = select; + select->next = branch; + branch->op = OP_jump; + branch->next_bb = arms[0][join_at[0]]; + } +} + +/* Whether rodata, data and function addresses are ADR's pc-relative offsets, + * settled before offsets are planned since it changes their length. + */ +static bool a64_pc_relative; + +/* Load the absolute address @target into @d. */ +static void a64_address(int d, int target) +{ + if (!a64_pc_relative) { + a64_mov_imm(d, target); + return; + } + int disp = target - (elf_code_start + elf_code->size); + a64_check_disp(disp, 21, "arm64 ADR target out of range"); + emit(a64_adr_insn(d, disp)); +} + +/* Whether the block @bb ends with a branch to @target. */ +static bool a64_branches_to(basic_block_t *bb, basic_block_t *target) +{ + vir_mop_t *tail = bb->mops.tail; + if (!tail) + return false; + if (tail->op == OP_jump) + return !tail->src2 && tail->next_bb == target; + return tail->op == OP_branch && + ((tail->src2 != 2 && tail->then_bb == target) || + (tail->src2 != 1 && tail->else_bb == target)); +} + +/* Where the loop body starting at code offset @offset in block @start goes: on + * the next boundary of 32 or 64 bytes if the body, from @start through the last + * branch back to it, fits one fetch block but would cross one where it is. Code + * starts after the headers, so addresses differ from offsets. Only an innermost + * loop without calls is placed: a call or an inner loop dominates the time of + * the body around it, and the padding would buy nothing. + */ +static int a64_place_loop(basic_block_t *start, int offset) +{ + int bytes = 0, length = 0; + bool inner = false; + for (basic_block_t *bb = start; bb && bytes <= A64_FETCH_BLOCK; + bb = bb->rpo_next) { + if (bb != start && bb->is_loop_start) + inner = true; + for (vir_mop_t *mop = bb->mops.head; mop; mop = mop->next) { + if (mop->op == OP_call || mop->op == OP_indirect) + inner = true; + if (mop->op != OP_allocat) + bytes += mop_count_words(mop) * 4; + } + if (a64_branches_to(bb, start) && bytes <= A64_FETCH_BLOCK && !inner) + length = bytes; + } + int address = elf_header_size() + offset; + if (!length || address % A64_FETCH_BLOCK + length <= A64_FETCH_BLOCK) + return offset; + int boundary = + length <= A64_FETCH_BLOCK / 2 ? A64_FETCH_BLOCK / 2 : A64_FETCH_BLOCK; + int padding = ALIGN_UP(address, boundary) - address; + return padding <= A64_LOOP_PADDING ? offset + padding : offset; +} + +static void a64_emit_entry(void); +static void a64_emit_main_call(void); +static void a64_emit_syscall(void); + +/* The instruction words @emitter produces, measured without writing them. */ +static int a64_count_words(void (*emitter)(void)) +{ + bool saved_count_only = mop_count_only; + int saved_counted_words = mop_counted_words; + + mop_count_only = true; + mop_counted_words = 0; + emitter(); + int words = mop_counted_words; + mop_count_only = saved_count_only; + mop_counted_words = saved_counted_words; + return words; +} + +/* Only a static program that calls __syscall needs the helper behind it; one + * built with --no-libc usually never does. */ -int a64_load_access(ph2_ir_t *p) +static bool a64_needs_syscall(void) { - return p->is_unsigned ? A64_LOAD_ZEXT : A64_LOAD; + func_t *f = find_func("__syscall"); + return !dynlink && f && f->bbs && f->is_used; } -/* AArch64's ordinary LDR/STR immediate is scaled by the access width. The - * compiler deliberately supports packed C layouts, so structure members are - * often not naturally aligned (func_t.bbs, for example). Such offsets must use - * the byte-addressed LDUR/STUR form; rounding them down corrupts adjacent - * fields during self-hosting. +/* Give the entry sequence, the global initializers and every block their code + * offsets, from the length of what emit_mop() produces for each operation. + * cfg_flatten() has already put each function's definition in MOP_FLAT ahead of + * its operations. */ -void a64_mem(int access, int size, int rt, int rn, int ofs) +static void a64_plan_offsets(void) { - if (size != 1 && size != 2 && size != 4 && size != 8) - fatal("unsupported arm64 access width"); - if (ofs >= -256 && ofs <= 255 && (ofs < 0 || ofs % size)) { - emit(a64_mem_unscaled_insn(access, size, rt, rn, ofs)); - return; + func_t *f; + int index = 0; + + /* The entry sequence, the global initializers, the call to main and the + * __syscall helper, in code_generate()'s order. + */ + elf_offset = a64_count_words(a64_emit_entry) * 4; + GLOBAL_FUNC->bbs->elf_offset = elf_offset; + mop_visit_globals(update_elf_offset); + if (MAIN_BB) + elf_offset += a64_count_words(a64_emit_main_call) * 4; + if (a64_needs_syscall()) { + find_func("__syscall")->bbs->elf_offset = elf_offset; + elf_offset += a64_count_words(a64_emit_syscall) * 4; } - if (ofs < 0 || ofs > 4095 * size || ofs % size) { - a64_mov_imm(A64_IP0, ofs); - a64_add(A64_IP0, rn, A64_IP0); - rn = A64_IP0; - ofs = 0; + for (f = FUNC_LIST.head; f; f = f->next) { + if (!f->bbs) + continue; + vir_mop_t *define = MOP_FLAT[index++]; + if (define->op != OP_define) + fatal("arm64 offset planning lost a function definition"); + + /* Every return after the first branches to its epilogue, which starts + * once the first has moved its own value into x0. Only a long epilogue + * is shared: in place of two restores and a RET, a taken branch on + * every early exit, the base case of a recursion among them, costs more + * than it saves. All returns of a function restore the same frame. + */ + vir_mop_t *first_return = NULL; + int exit_offset = 0; + for (basic_block_t *bb = f->bbs; bb; bb = bb->rpo_next) { + if (bb->is_loop_start) + elf_offset = a64_place_loop(bb, elf_offset); + bb->elf_offset = elf_offset; + if (bb == f->bbs) + elf_offset += mop_count_words(define) * 4; + for (vir_mop_t *mop = bb->mops.head; mop; mop = mop->next) { + index++; + if (mop->op == OP_return) { + int moves = mop->src0 >= 0 && a64_reg(mop->src0) != 0; + mop->src2 &= ~A64_SHARED_EXIT; + bool long_exit = + mop_count_words(mop) - moves >= A64_SHARED_EXIT_WORDS; + if (long_exit && !first_return) { + first_return = mop; + exit_offset = elf_offset + moves * 4; + } else if (long_exit) { + mop->src2 |= A64_SHARED_EXIT; + mop->dest = exit_offset; + } + } + update_elf_offset(mop); + } + } } - emit(a64_mem_scaled_insn(access, size, rt, rn, ofs)); } -/* Masking an out-of-range displacement silently branches somewhere else. The - * generated image is small enough that these never fire today, so say so rather - * than let a larger input fail as a wild jump. +/* Mark the blocks a branch from later in the layout reaches: with the loop + * tests rotated, these are where loop bodies start. Only an emitted branch + * counts, not a successor that the code falls into. */ -void a64_check_disp(int disp, int bits, char *what) +static void a64_mark_loop_starts(func_t *f) { - int lim = 1 << (bits - 1); - if (disp < -lim || disp >= lim) - fatal(what); + for (basic_block_t *bb = f->bbs; bb; bb = bb->rpo_next) + bb->is_loop_start = false; + for (basic_block_t *bb = f->bbs; bb; bb = bb->rpo_next) { + vir_mop_t *tail = bb->mops.tail; + basic_block_t *targets[2] = {NULL, NULL}; + if (!tail) + continue; + if (tail->op == OP_jump && !tail->src2) + targets[0] = tail->next_bb; + else if (tail->op == OP_branch) { + if (tail->src2 != 2) + targets[0] = tail->then_bb; + if (tail->src2 != 1) + targets[1] = tail->else_bb; + } + for (int i = 0; i < 2; i++) + if (targets[i] && targets[i] != f->bbs && + targets[i]->rpo <= bb->rpo) + targets[i]->is_loop_start = true; + } } -/* Branch to @target when @rt is non-zero. CBNZ carries the same 19-bit - * displacement a B.cond would, so it replaces a compare-and-branch pair - * outright. @wide picks the X form, which a pointer needs: testing only Wn - * reads an address whose low word happens to be zero as null. +/* Whether @bb ends with a branch, jump or return, which emit_mop() points at + * whatever block then follows. Any other block falls into the next one. */ -void a64_cbnz(bool wide, int rt, int target) -{ - int disp = (target - elf_code->size) / 4; - a64_check_disp(disp, 19, "arm64 conditional branch out of range"); - emit(a64_cbnz_insn(wide, rt, disp)); -} -void a64_b(int target) -{ - int d = (target - elf_code->size) / 4; - a64_check_disp(d, 26, "arm64 branch out of range"); - emit(a64_b_insn(d)); -} -void a64_bl_addr(int target) +static bool a64_ends_closed(basic_block_t *bb) { - int d = (target - (elf_code_start + elf_code->size)) / 4; - a64_check_disp(d, 26, "arm64 call out of range"); - emit(a64_bl_insn(d)); + vir_mop_t *tail = bb->mops.tail; + return tail && (tail->op == OP_return || tail->op == OP_jump || + tail->op == OP_branch); } -int a64_adrp_insn(int d, int pc, int target) + +/* Whether @bb ends with a branch or jump to @target. */ +static bool a64_targets(basic_block_t *bb, basic_block_t *target) { - int pages = (target >> 12) - (pc >> 12); - a64_check_disp(pages, 21, "arm64 ADRP target out of range"); - return a64_adrp_pages_insn(d, pages); + vir_mop_t *tail = bb->mops.tail; + if (tail->op == OP_jump) + return tail->next_bb == target; + return tail->op == OP_branch && + (tail->then_bb == target || tail->else_bb == target); } -/* Which operand of an address expression is the int index that must widen: 0 - * for neither, 1 for src0, 2 for src1. Exactly one source being an address is - * what makes the other an index, so the two source flags decide alone. - */ -int a64_ptr_index(ph2_ir_t *p) +/* The last block from @first up to @end that emits anything, or NULL. */ +static basic_block_t *a64_last_emitting(basic_block_t *first, + basic_block_t *end) { - if (p->src0_is_pointer == p->src1_is_pointer) - return 0; - return p->src0_is_pointer ? 2 : 1; + basic_block_t *last = NULL; + for (basic_block_t *bb = first; bb != end; bb = bb->rpo_next) + if (bb->mops.head) + last = bb; + return last; } -/* A comparison is as wide as the values it compares. An address has to be - * compared whole, an array included, since what reaches the instruction is its - * decayed base. The source flags say so and is_pointer does not: it counts - * pointer-like operands alone, for the sake of the 32-bit targets that read it. +/* What each register holds where @bb ends, as a symbol plus an offset. Symbol 0 + * is a literal, set in @bb or kept in a lane. Any other symbol stands for a + * value that is unknown but the same wherever it appears, such as what a + * register held on entry to @bb or the base of the globals, so that two + * registers with one symbol differ by the difference of their offsets: a + * pointer and the end of the array it walks, say. */ -bool a64_cmp_wide(ph2_ir_t *p) +static void a64_values_at_end(basic_block_t *bb, + int *symbol, + unsigned long long *offset) { - return p->size_bytes == 8 || p->src0_is_pointer || p->src1_is_pointer; + int fresh = REG_CNT + 1; + for (int r = 0; r < REG_CNT; r++) { + bool lane = (a64_lanes & (1u << r)) != 0; + symbol[r] = lane ? 0 : r + 1; + offset[r] = lane ? a64_lane_values[r] : 0; + } + for (vir_mop_t *ir = bb->mops.head; ir; ir = ir->next) { + int d = ir->dest, n = ir->src0; + unsigned int defs = mop_defs(ir); + if (d < 0 || d >= REG_CNT || defs != 1u << d) { + for (int r = 0; r < REG_CNT; r++) + if (defs & (1u << r)) { + symbol[r] = ++fresh; + offset[r] = 0; + } + continue; + } + bool copies = n >= 0 && n < REG_CNT; + if (ir->op == OP_load_constant) { + symbol[d] = 0; + offset[d] = a64_constant_value(ir); + } else if (ir->op == OP_assign && copies) { + symbol[d] = symbol[n]; + offset[d] = offset[n]; + } else if ((ir->op == OP_add || ir->op == OP_sub) && copies && + ir->size_bytes == 8 && ir->src2 > 0) { + unsigned long long imm = (unsigned long long) a64_imm12(ir) + << (ir->src2 > A64_IMM_SHIFTED ? 12 : 0); + symbol[d] = symbol[n]; + offset[d] = ir->op == OP_add ? offset[n] + imm : offset[n] - imm; + } else if (ir->op == OP_global_address_of) { + symbol[d] = REG_CNT + 1; + offset[d] = ir->src0; + } else { + symbol[d] = ++fresh; + offset[d] = 0; + } + } } -a64_cond_t a64_cond(opcode_t op, bool is_unsigned) +/* Which way the test that is all of @bb goes when entered from @from: 1 to its + * then block, 0 to its else block, -1 when that is not known there. Literals + * settle any test; two values of one symbol settle only an equality test, as + * the order of unknown values is not known even when their distance is. + */ +static int a64_known_branch(basic_block_t *from, basic_block_t *bb) { - switch (op) { + vir_mop_t *branch = bb->mops.tail, *compare = bb->mops.head; + int symbol[REG_CNT]; + unsigned long long offset[REG_CNT], a, b; + if (!branch || branch->op != OP_branch) + return -1; + a64_values_at_end(from, symbol, offset); + if (compare == branch) { + int r = branch->src0; + if (branch->src1 || r < 0 || r >= REG_CNT || symbol[r]) + return -1; + a = offset[r]; + if (!branch->src0_is_pointer && branch->size_bytes != 8) + a = (unsigned int) a; + return a != 0; + } + int ra = compare->src0, rb = compare->src1; + if (compare->next != branch || !op_is_comparison(compare->op) || + !(compare->src2 & A64_CMP_FUSED) || (compare->src2 & A64_CMP_TESTED) || + ra < 0 || ra >= REG_CNT) + return -1; + int sa = symbol[ra], sb = 0; + a = offset[ra]; + if (compare->src2 >> 1) + b = (long long) ((compare->src2 >> 1) - A64_CMP_BIAS); + else if (rb >= 0 && rb < REG_CNT) { + sb = symbol[rb]; + b = offset[rb]; + } else + return -1; + if (sa != sb) + return -1; + bool is_unsigned = compare->src0_is_unsigned || compare->src1_is_unsigned; + if (!a64_cmp_wide(compare)) { + a = is_unsigned ? (unsigned int) a : (long long) (int) a; + b = is_unsigned ? (unsigned int) b : (long long) (int) b; + } + if (sa) + return compare->op == OP_eq ? a == b + : compare->op == OP_neq ? a != b + : -1; + bool less = is_unsigned ? a < b : (long long) a < (long long) b; + switch (compare->op) { case OP_eq: - return A64_EQ; + return a == b; case OP_neq: - return A64_NE; - case OP_geq: - return is_unsigned ? A64_HS : A64_GE; + return a != b; case OP_lt: - return is_unsigned ? A64_LO : A64_LT; + return less; + case OP_geq: + return !less; case OP_gt: - return is_unsigned ? A64_HI : A64_GT; + return !less && a != b; + case OP_leq: + return less || a == b; default: - return is_unsigned ? A64_LS : A64_LE; + return -1; } } -/* All estimates below exactly match emit_ph2_ir(). Fixed-width AArch64 makes - * this deliberately simpler and more reliable than a relocation pass. +/* A rotated loop keeps its first test at the entry, where the counter and the + * bound are often both literals, as in a for loop over a constant range. That + * test is settled now: it becomes a jump to the body. */ -void update_elf_offset(ph2_ir_t *ir) +static void a64_settle_entry_tests(func_t *f) { - int n = 1; - switch (ir->op) { - case OP_allocat: - n = 0; - break; - case OP_assign: - n = ir->dest == ir->src0 ? 0 : 1; - break; - case OP_load_constant: - case OP_load_data_address: - case OP_load_rodata_address: - n = ir->op == OP_load_constant && ir->size_bytes == 8 ? 4 : 2; - break; - case OP_address_of: - case OP_global_address_of: - n = 3; - break; - case OP_load: - case OP_global_load: - n = a64_mem_count(ir->size_bytes, ir->src0); - break; - case OP_store: - case OP_global_store: - n = a64_mem_count(ir->size_bytes, ir->src1); - break; - case OP_address_of_func: - n = 3; - break; - case OP_return: - n = (ir->src0 >= 0 && a64_reg(ir->src0) != 0) ? 8 : 7; - break; - case OP_branch: - n = 2; - break; - case OP_mod: - n = 2; - break; - case OP_eq: - case OP_neq: - case OP_gt: - case OP_lt: - case OP_geq: - case OP_leq: - case OP_log_not: - n = 2; - break; - default: - break; + for (basic_block_t *bb = f->bbs; bb; bb = bb->rpo_next) { + if (bb == f->bbs || a64_branches_naming(f, bb) != 1) + continue; + basic_block_t *from = NULL, *layout = a64_last_emitting(f->bbs, bb); + for (basic_block_t *p = f->bbs; p; p = p->rpo_next) + if (p->mops.tail && p->mops.tail->op == OP_jump && + p->mops.tail->next_bb == bb) + from = p; + if (!from || (layout != from && !a64_ends_closed(layout))) + continue; + int taken = a64_known_branch(from, bb); + if (taken < 0) + continue; + vir_mop_t *jump = bb->mops.tail; + jump->op = OP_jump; + jump->next_bb = taken ? jump->then_bb : jump->else_bb; + bb->mops.head = jump; } - elf_offset += n * 4; } -void cfg_flatten(void) +/* A rotated loop tests again at its end and leaves for the code after it, which + * the layout put between the entry test and the body: the body then ends with a + * jump there. Moving that code behind the body lets the last test fall into it + * and the entry test into the body, which saves the jump. It is not moved when + * its own end falls into the body, which would cost a jump instead. + */ +static void a64_place_exits(func_t *f) { - func_t *f; + for (basic_block_t *latch = f->bbs; latch; latch = latch->rpo_next) { + vir_mop_t *tail = latch->mops.tail; + if (!tail || tail->op != OP_branch) + continue; + basic_block_t *body = tail->then_bb, *exit = tail->else_bb; + if (body->rpo > latch->rpo) { + body = tail->else_bb; + exit = tail->then_bb; + } + if (body->rpo > latch->rpo || exit->rpo >= body->rpo || exit == f->bbs) + continue; + basic_block_t *before = f->bbs; + while (before->rpo_next != exit) + before = before->rpo_next; + basic_block_t *last = exit; + while (last->rpo_next != body) + last = last->rpo_next; + basic_block_t *open = a64_last_emitting(exit, body); + if (!a64_ends_closed(before) || !open || !a64_ends_closed(open) || + a64_targets(open, body)) + continue; + before->rpo_next = body; + last->rpo_next = latch->rpo_next; + latch->rpo_next = exit; + int rpo = 0; + for (basic_block_t *bb = f->bbs; bb; bb = bb->rpo_next) + bb->rpo = rpo++; + } + for (basic_block_t *bb = f->bbs; bb; bb = bb->rpo_next) { + vir_mop_t *ir = bb->mops.tail; + if (ir && ir->op == OP_jump) + ir->src2 = a64_falls_into(bb, ir->next_bb); + else if (ir && ir->op == OP_branch) + ir->src2 = a64_branch_fallthrough(bb, ir); + } +} - /* Entry sequence lengths, which the block offsets below start after. - * Static: 12 instructions of setup, then the 9-instruction __syscall - * helper, so 21 in all. Dynamic: 25, having no __syscall helper but saving - * the original stack pointer for __libc_start_main's stack_end argument, - * spilling argc/argv, and clearing the frame with an inline loop. - */ - f = find_func("__syscall"); - if (f && f->bbs) { - if (dynlink) - f->bbs->elf_offset = 0; - else - f->bbs->elf_offset = 12 * 4; +/* A load from a base register that an addition of a small literal to that + * register follows, with nothing between that reads or writes it, becomes one + * load that writes the sum back: LDR Wt, [Xn], #imm, as a pointer walking an + * array does. The written-back base goes in the load's dest_hi, so that its + * register counts as defined, and the amount in src2. This runs after every + * other pass, which all leave an operation with a dest_hi alone. + */ +static void a64_post_index(func_t *f) +{ + for (basic_block_t *bb = f->bbs; bb; bb = bb->rpo_next) + for (vir_mop_t *load = bb->mops.head; load; load = load->next) { + int base = load->src0; + if (load->op != OP_read || load->src0_hi >= 0 || load->src2 || + load->dest_hi >= 0 || base < 0 || base >= REG_CNT || + load->dest == base) + continue; + vir_mop_t *before = load, *step = load->next; + bool found = false; + for (int scan = 0; step && scan < 4 && !found; scan++) { + found = (step->op == OP_add || step->op == OP_sub) && + step->dest == base && step->src0 == base && + step->size_bytes == 8 && step->src2 > 0 && + step->src2 <= A64_IMM_SHIFTED && step->dest_hi < 0; + if (found) + break; + if (ir_reads_reg(step, base) || + (mop_defs(step) & (1u << base)) || step->op == OP_branch || + step->op == OP_jump || step->op == OP_return) + break; + before = step; + step = step->next; + } + if (!found) + continue; + int amount = a64_imm12(step); + if (step->op == OP_sub) + amount = -amount; + if (amount < -256 || amount > 255) + continue; + load->dest_hi = base; + load->src2 = amount; + before->next = step->next; + if (bb->mops.tail == step) + bb->mops.tail = before; + } +} + +/* An int extended to 64 bits and then shifted left by a literal is one SBFIZ, + * or UBFIZ for an unsigned one, as when an index scales to a byte offset. The + * shift keeps the literal in src2 and names the extension's source in src0; + * src1, which no register number matches, says which of the two it is. This + * runs after the folds that would take the shift into an addition. + */ +#define A64_SBFIZ (-2) +#define A64_UBFIZ (-3) + +static void a64_insert_fields(func_t *f) +{ + for (basic_block_t *bb = f->bbs; bb; bb = bb->rpo_next) + for (vir_mop_t **link = &bb->mops.head; *link && (*link)->next;) { + vir_mop_t *extend = *link, *shift = extend->next; + if (extend->op != OP_sign_ext || extend->src1 != ((4 << 16) | 8) || + extend->dest_hi >= 0 || extend->src0 < 0 || + extend->src0 >= REG_CNT || shift->op != OP_lshift || + shift->size_bytes != 8 || shift->src2 < 2 || shift->src2 > 33 || + shift->src0 != extend->dest || shift->dest_hi >= 0 || + !a64_value_dies(bb, extend, shift)) { + link = &extend->next; + continue; + } + shift->src0 = extend->src0; + shift->src1 = extend->src0_is_unsigned ? A64_UBFIZ : A64_SBFIZ; + *link = shift; + } +} + +/* Early returns that run ahead of the frame, as fib's base case does. When @f + * starts with nothing but copies of its arguments, then a test of them whose + * one side only returns an argument or nothing, that test and its return need + * no saved register, so they leave @f's blocks for its prologue, which starts + * with the comparison, a branch past the return, and the return. Recorded in + * @f's early_test, early_cond and early_value. + */ +static void a64_early_exit(func_t *f) +{ + basic_block_t *entry = f->bbs; + vir_mop_t *branch = entry->mops.tail, *compare = NULL; + int argument[REG_CNT]; + for (int reg = 0; reg < REG_CNT; reg++) + argument[reg] = reg < MAX_ARGS_IN_REG ? reg : -1; + if (!branch || branch->op != OP_branch || !branch->src1) + return; + for (vir_mop_t *ir = entry->mops.head; ir != branch; ir = ir->next) { + if (ir->next == branch && op_is_comparison(ir->op) && + (ir->src2 & A64_CMP_FUSED) && !(ir->src2 & A64_CMP_TESTED) && + ir->src0_hi < 0 && ir->src1_hi < 0) { + compare = ir; + break; + } + if (ir->op != OP_assign || ir->dest < 0 || ir->dest >= REG_CNT || + ir->src0 < 0 || ir->src0 >= REG_CNT || ir->dest_hi >= 0) + return; + argument[ir->dest] = argument[ir->src0]; } - if (dynlink) - elf_offset = 25 * 4; - else - elf_offset = 21 * 4; - GLOBAL_FUNC->bbs->elf_offset = elf_offset; - for (ph2_ir_t *p = GLOBAL_FUNC->bbs->ph2_ir_list.head; p; p = p->next) - update_elf_offset(p); + if (!compare || compare->src0 < 0 || compare->src0 >= REG_CNT || + argument[compare->src0] < 0 || + (!(compare->src2 >> 1) && + (compare->src1 < 0 || compare->src1 >= REG_CNT || + argument[compare->src1] < 0))) + return; - /* code_generate() emits the call to main only when there is one, so a - * translation unit without main must not be charged for it either. - */ - if (MAIN_BB) { - int global_frame = ALIGN_UP(GLOBAL_FUNC->stack_size, A64_STACK_ALIGN); - if (dynlink) - elf_offset += (10 + a64_mem_count(8, global_frame) + - a64_mem_count(8, global_frame + 8)) * - 4; - else - elf_offset += 6 * 4; + /* The side that only returns, reached from nowhere else. */ + basic_block_t *exit = NULL, *rest = NULL; + for (int side = 0; side < 2 && !exit; side++) { + basic_block_t *bb = side ? branch->else_bb : branch->then_bb; + vir_mop_t *ret = bb ? bb->mops.head : NULL; + basic_block_t *before = bb ? a64_last_emitting(f->bbs, bb) : NULL; + if (ret && ret->op == OP_return && !ret->next && + !(ret->src2 & A64_SHARED_EXIT) && ret->src0_hi < 0 && + (ret->src0 < 0 || + (ret->src0 < REG_CNT && argument[ret->src0] >= 0)) && + a64_branches_naming(f, bb) == 1 && + (before == entry || (before && a64_ends_closed(before)))) { + exit = bb; + rest = side ? branch->then_bb : branch->else_bb; + } } + if (!exit || !rest) + return; + + vir_mop_t *test = mop_create(compare->op); + *test = *compare; + test->next = NULL; + test->src0 = argument[compare->src0]; + if (!(compare->src2 >> 1)) + test->src1 = argument[compare->src1]; + vir_mop_t *ret = exit->mops.head; + f->early_test = test; + f->early_cond = (branch->src1 - 1) ^ (exit == branch->else_bb); + f->early_value = ret->src0 < 0 ? -1 : argument[ret->src0]; + + /* The entry block now only copies and goes on with the rest. */ + vir_mop_t **link = &entry->mops.head; + while (*link != compare) + link = &(*link)->next; + vir_mop_t *jump = mop_create(OP_jump); + jump->next_bb = rest; + *link = jump; + entry->mops.tail = jump; + entry->then_ = entry->else_ = NULL; + entry->next = rest; + jump->src2 = a64_falls_into(entry, rest); + exit->mops.head = exit->mops.tail = NULL; +} + +void cfg_flatten(void) +{ + func_t *f; + for (f = FUNC_LIST.head; f; f = f->next) + if (f->bbs) { + a64_merge_blocks(f); + arm64_fold_immediates(f); + a64_coalesce_results(f); + a64_fuse_multiply_add(f); + a64_drop_dead(f); + a64_hoist_constants(f); + a64_select_diamonds(f); + for (basic_block_t *bb = f->bbs; bb; bb = bb->rpo_next) + for (vir_mop_t **link = &bb->mops.head; *link; + link = &(*link)->next) { + vir_mop_t *ir = *link, *next = ir->next; + if (a64_test_bits(bb, ir)) + continue; + if (a64_branch_only_compare(bb, ir)) { + if (a64_branch_on_zero(ir, next)) + *link = ir = next; + else { + ir->src2 |= A64_CMP_FUSED; + next->src1 = + a64_cond(ir->op, ir->src0_is_unsigned || + ir->src1_is_unsigned) + + 1; + } + } + if (ir->op == OP_jump) + ir->src2 = a64_falls_into(bb, ir->next_bb); + else if (ir->op == OP_branch) + ir->src2 = a64_branch_fallthrough(bb, ir); + } + + /* Repeat a small loop test at its back edge so the body needs only + * one taken branch per iteration. Keep the original comparison + * flags; only the copied branch's physical fallthrough changes. + */ + for (basic_block_t *bb = f->bbs; bb; bb = bb->rpo_next) { + vir_mop_t *jump = bb->mops.tail; + if (!jump || jump->op != OP_jump || !jump->next_bb || + jump->next_bb->rpo > bb->rpo) + continue; + basic_block_t *head = jump->next_bb; + int count = 0; + for (vir_mop_t *ir = head->mops.head; ir; ir = ir->next) + count += ir->op == OP_load || ir->op == OP_load_constant || + op_is_comparison(ir->op) || + ir->op == OP_branch + ? 1 + : 4; + if (!count || count > 3 || head->mops.tail->op != OP_branch) + continue; + for (vir_mop_t *ir = head->mops.head; ir; ir = ir->next) { + *jump = *ir; + jump->next = ir->next ? mop_create(ir->next->op) : NULL; + bb->mops.tail = jump; + if (jump->next) + jump = jump->next; + } + jump->src2 = a64_branch_fallthrough(bb, jump); + } + a64_settle_entry_tests(f); + a64_place_exits(f); + a64_post_index(f); + a64_insert_fields(f); + } + mop_flat_prepare(false); for (f = FUNC_LIST.head; f; f = f->next) { if (!f->bbs) continue; - ph2_ir_t *d = add_ph2_ir(OP_define); + vir_mop_t *d = add_mop(OP_define); d->src0 = f->stack_size; d->func_name = intern_string(f->return_def.var_name); + int saved = + func_highest_used_reg(f, A64_FIRST_CALLEE) - (A64_FIRST_CALLEE - 1); + if (dynlink && !strcmp(d->func_name, "main")) + saved |= A64_SAVE_GP; + if (!saved && func_is_frameless_leaf(f)) + saved |= A64_FRAMELESS; + int locals = ALIGN_UP(f->stack_size, A64_STACK_ALIGN); + if (!(saved & A64_FRAMELESS) && locals && + locals <= A64_FOLDED_FRAME_MAX && !f->passes_stack_args) { + saved |= A64_FOLDED_FRAME; + for (basic_block_t *bb = f->bbs; bb; bb = bb->rpo_next) + for (vir_mop_t *ir = bb->mops.head; ir; ir = ir->next) { + if (ir->ofs_based_on_stack_top) + continue; + if (ir->op == OP_load || ir->op == OP_address_of) + ir->src0 += 16; + else if (ir->op == OP_store) + ir->src1 += 16; + } + } + d->src2 = saved; + if (!(saved & (A64_FRAMELESS | A64_SAVE_GP))) + a64_early_exit(f); /* Where the caller's stack arguments sit, seen from the callee's own * SP. This must round the frame exactly as the prologue does: rounding @@ -371,54 +2139,76 @@ void cfg_flatten(void) * one slot low. */ int stack_top_ofs = - ALIGN_UP(f->stack_size, A64_STACK_ALIGN) + A64_SAVE_BYTES; - for (basic_block_t *b = f->bbs; b; b = b->rpo_next) { - b->elf_offset = elf_offset; + ALIGN_UP(f->stack_size, A64_STACK_ALIGN) + a64_save_bytes(saved); + mop_flatten_function(f, stack_top_ofs, 0, saved, true, NULL); + } - /* The entry block's offset deliberately names the prologue so calls - * enter at a valid function entry. Later block labels must however - * account for that prologue before their first IR word. - */ - if (b == f->bbs) { - elf_offset += 7 * 4; - if (dynlink && !strcmp(f->return_def.var_name, "main")) - elf_offset += 3 * 4; - } - for (ph2_ir_t *p = b->ph2_ir_list.head; p; p = p->next) { - if (p->ofs_based_on_stack_top) { - if (p->op == OP_load || p->op == OP_address_of) - p->src0 += stack_top_ofs; - else if (p->op == OP_store) - p->src1 += stack_top_ofs; - } - ph2_ir_t *q = add_existed_ph2_ir(p); - if (q->op == OP_return) - q->src1 = f->stack_size; - update_elf_offset(q); - } - } + /* ADR reaches 1 MiB either way. A static image keeps .rodata after the code + * and .data on the page after that, so when the whole span is in reach, + * every address the code takes is one instruction; otherwise plan again + * with absolute addresses. + */ + for (f = FUNC_LIST.head; f; f = f->next) + if (f->bbs) + a64_mark_loop_starts(f); + a64_pc_relative = !dynlink; + a64_plan_offsets(); + if (a64_pc_relative && + elf_offset + elf_rodata->size + elf_data->size + 2 * PAGESIZE >= + 1 << 20) { + a64_pc_relative = false; + a64_plan_offsets(); } } -void emit_ph2_ir(ph2_ir_t *p) +void emit_mop(vir_mop_t *p) { int d = a64_reg(p->dest), n = a64_reg(p->src0), m = a64_reg(p->src1); + if (op_is_comparison(p->op)) { + int imm = (p->src2 >> 1) - A64_CMP_BIAS; + if (p->src2 & A64_CMP_TESTED) + ; + else if (p->src2 >> 1 == 0) + emit(a64_cmp_reg_insn(a64_cmp_wide(p), n, m)); + else if (imm < 0) + emit(a64_cmn_imm_insn(a64_cmp_wide(p), n, -imm)); + else + emit(a64_cmp_imm_insn(a64_cmp_wide(p), n, imm)); + if (!(p->src2 & A64_CMP_FUSED)) + emit(a64_cset_insn( + a64_cmp_wide(p), d, + a64_cond(p->op, p->src0_is_unsigned || p->src1_is_unsigned))); + return; + } switch (p->op) { case OP_define: { - bool reload_global_base = dynlink && !strcmp(p->func_name, "main"); - fatal_function_context = p->func_name; - emit(a64_stp_pre_insn(A64_FP, A64_LR, A64_SP, -16)); + if (p->src2 & A64_FRAMELESS) + return; + func_t *func = find_func(p->func_name); + if (func && func->early_test) { + /* See a64_early_exit(): the test, a branch past the return when it + * fails, the argument into x0 and the return. + */ + int value = func->early_value, words = value > 0 ? 2 : 1; + emit_mop(func->early_test); + emit(a64_b_cond_insn(func->early_cond ^ 1, words + 1)); + if (value > 0) + a64_mov(0, value); + emit(a64_ret_insn()); + } + int saves = (p->src2 & 15) + !!(p->src2 & A64_SAVE_GP); + int locals = ALIGN_UP(p->src0, A64_STACK_ALIGN); + bool folded = p->src2 & A64_FOLDED_FRAME; + emit(a64_stp_pre_insn(A64_FP, A64_LR, A64_SP, + folded && !saves ? -16 - locals : -16)); emit(a64_add_imm_insn(true, A64_FP, A64_SP, 0)); - emit(a64_stp_pre_insn(20, 21, A64_SP, -16)); - - /* x19 holds the synthetic global-frame base, but AAPCS64 makes it - * callee-saved and glibc calls into this code at main. Saving it - * alongside x22 costs nothing: the slot was half empty anyway. - */ - emit(a64_stp_pre_insn(22, A64_GP, A64_SP, -16)); - a64_mov_imm(A64_IP0, ALIGN_UP(p->src0, A64_STACK_ALIGN)); - a64_sub(A64_SP, A64_SP, A64_IP0); + for (int reg = 0; reg < saves; reg += 2) + emit(a64_stp_pre_insn( + a64_saved_reg(p->src2, reg), a64_saved_reg(p->src2, reg + 1), + A64_SP, folded && reg + 2 >= saves ? -16 - locals : -16)); + if (!folded) + a64_adjust_frame(locals, true); /* Reload x19 rather than trust what called us: glibc enters at main, * and AAPCS64 lets everything in between clobber a callee-saved @@ -427,32 +2217,40 @@ void emit_ph2_ir(ph2_ir_t *p) * itself is allocated once in code_generate(), which is also where the * question of clearing it is settled. */ - if (reload_global_base) { + if (p->src2 & A64_SAVE_GP) { a64_mov_imm(A64_IP0, elf_data_start); a64_mem(A64_LOAD, 8, A64_GP, A64_IP0, 0); } return; } case OP_load_constant: - if (p->size_bytes == 8) - a64_mov_imm_wide(d, p->src0, p->src1); - else if (p->is_unsigned) - a64_mov_imm_unsigned(d, p->src0); - else - a64_mov_imm(d, p->src0); + a64_load_constant(d, p); return; case OP_assign: if (d != n) a64_mov(d, n); return; case OP_address_of: + case OP_global_address_of: { + /* Both offsets are final when the size estimate runs, so a 12-bit one + * is an immediate of the ADD itself. + */ + int base = p->op == OP_address_of ? A64_SP : A64_GP; + if (p->src0 >= 0 && p->src0 < 4096) { + emit(a64_add_imm_insn(true, d, base, p->src0)); + return; + } + if (p->src0 > 0 && p->src0 < 1 << 24) { + emit(a64_add_imm_insn(true, d, base, p->src0 >> 12) | + A64_IMM_LSL12); + if (p->src0 & 0xfff) + emit(a64_add_imm_insn(true, d, d, p->src0 & 0xfff)); + return; + } a64_mov_imm(A64_IP0, p->src0); - a64_add(d, A64_SP, A64_IP0); - return; - case OP_global_address_of: - a64_mov_imm(A64_IP0, p->src0); - a64_add(d, A64_GP, A64_IP0); + a64_add(d, base, A64_IP0); return; + } case OP_load: a64_mem(a64_load_access(p), p->size_bytes, d, A64_SP, p->src0); return; @@ -466,40 +2264,49 @@ void emit_ph2_ir(ph2_ir_t *p) a64_mem(A64_STORE, p->size_bytes, n, A64_GP, p->src1); return; case OP_read: - a64_mem(a64_load_access(p), p->src1, d, n, 0); - return; - case OP_write: - a64_mem(A64_STORE, p->dest, m, n, 0); + case OP_write: { + bool read = p->op == OP_read; + int access = read ? a64_load_access(p) : A64_STORE; + int size = read ? p->src1 : p->dest; + int rt = read ? d : m; + if (read && p->dest_hi >= 0) + emit(a64_mem_post_insn(access, size, rt, n, p->src2)); + else if (p->src0_hi >= 0) + emit(a64_mem_register_insn( + access, size, rt, n, a64_reg(p->src0_hi), + p->src2 != -1 && p->src2 != -5 && p->src2 != -9, + p->src2 <= -5) | + (p->src2 <= -9 ? A64_SXTW_INDEX : 0)); + else + a64_mem(access, size, rt, n, p->src2); return; + } case OP_add: - /* Index expressions are int-valued, so the index has to widen before it - * joins a 64-bit address; otherwise -1 becomes +4294967295 on LP64. - * Addition is commutative, so either operand may be the index. - * - * The fall-through stays the X form for both cases a64_ptr_index() - * reports as 0: two addresses, which is pointer minus pointer and - * genuinely 64-bit, and two ints, whose upper half no reader looks at. - * See the width note at the comparisons below. - */ - if (a64_ptr_index(p) == 2) - a64_add_sxtw(d, n, m); - else if (a64_ptr_index(p) == 1) - a64_add_sxtw(d, m, n); - else + if (p->src2 < 0) + emit(a64_add_reg_insn(a64_arith_wide(p), d, n, m) | a64_shifted(p)); + else if (p->src2) + emit(a64_add_imm_insn(a64_arith_wide(p), d, n, a64_imm12(p)) | + (p->src2 > A64_IMM_SHIFTED ? A64_IMM_LSL12 : 0)); + else if (a64_arith_wide(p)) a64_add(d, n, m); + else + emit(a64_add_reg_insn(false, d, n, m)); return; case OP_sub: - /* Only pointer-minus-int widens: int-minus-pointer is not an address - * expression, and pointer-minus-pointer is already 64-bit on both - * sides. - */ - if (a64_ptr_index(p) == 2) - a64_sub_sxtw(d, n, m); - else + if (p->src2 < 0) + emit(a64_sub_reg_insn(a64_arith_wide(p), d, n, m) | a64_shifted(p)); + else if (p->src2) + emit(a64_sub_imm_insn(a64_arith_wide(p), d, n, a64_imm12(p)) | + (p->src2 > A64_IMM_SHIFTED ? A64_IMM_LSL12 : 0)); + else if (a64_arith_wide(p)) a64_sub(d, n, m); + else + emit(a64_sub_reg_insn(false, d, n, m)); return; case OP_mul: - emit(a64_mul_insn(p->size_bytes == 8, d, n, m)); + emit(p->src1_hi >= 0 ? a64_madd_insn(p->size_bytes == 8, d, n, m, + a64_reg(p->src1_hi)) + : a64_mul_insn(p->size_bytes == 8, d, n, m)); return; case OP_div: emit(a64_div_insn(p, d, n, m)); @@ -512,19 +2319,40 @@ void emit_ph2_ir(ph2_ir_t *p) emit(a64_msub_insn(p->size_bytes == 8, d, A64_IP0, m, n)); return; case OP_lshift: - emit(a64_lslv_insn(p->size_bytes == 8, d, n, m)); + if (p->src1 == A64_SBFIZ || p->src1 == A64_UBFIZ) { + /* SBFIZ or UBFIZ Xd, Xn, #s, #32: SBFM/UBFM with immr 64 - s. */ + int shift = p->src2 - 1; + emit((p->src1 == A64_UBFIZ ? 0xd3400000U : 0x93400000U) | + (((64 - shift) & 63) << 16) | (31 << 10) | (n << 5) | d); + return; + } + emit(p->src2 ? a64_lsl_imm_insn(p->size_bytes == 8, d, n, p->src2 - 1) + : a64_lslv_insn(p->size_bytes == 8, d, n, m)); return; case OP_rshift: emit(a64_rshift_insn(p, d, n, m)); return; - case OP_bit_and: - emit(a64_and_reg_insn(p->size_bytes == 8, d, n, m)); + case OP_bit_and: { + /* Into the zero register, it is ANDS for the comparison after it. */ + int flags = p->dest == A64_ZERO ? 0x60000000 : 0; + emit((p->src2 > 0 ? a64_logic_imm_insn(0x12000000, p->size_bytes == 8, + d, n, p->src2 - 1) + : a64_and_reg_insn(p->size_bytes == 8, d, n, m) | + a64_shifted(p)) | + flags); return; + } case OP_bit_or: - emit(a64_orr_reg_insn(p->size_bytes == 8, d, n, m)); + emit(p->src2 > 0 ? a64_logic_imm_insn(0x32000000, p->size_bytes == 8, d, + n, p->src2 - 1) + : a64_orr_reg_insn(p->size_bytes == 8, d, n, m) | + a64_shifted(p)); return; case OP_bit_xor: - emit(a64_eor_reg_insn(p->size_bytes == 8, d, n, m)); + emit(p->src2 > 0 ? a64_logic_imm_insn(0x52000000, p->size_bytes == 8, d, + n, p->src2 - 1) + : a64_eor_reg_insn(p->size_bytes == 8, d, n, m) | + a64_shifted(p)); return; case OP_negate: emit(a64_neg_insn(p->size_bytes == 8, d, n)); @@ -532,17 +2360,6 @@ void emit_ph2_ir(ph2_ir_t *p) case OP_bit_not: emit(a64_mvn_insn(p->size_bytes == 8, d, n)); return; - case OP_eq: - case OP_neq: - case OP_gt: - case OP_lt: - case OP_geq: - case OP_leq: - emit(a64_cmp_reg_insn(a64_cmp_wide(p), n, m)); - emit(a64_cset_insn( - a64_cmp_wide(p), d, - a64_cond(p->op, p->src0_is_unsigned || p->src1_is_unsigned))); - return; /* Width follows the operand, exactly as the comparisons above do. A pointer * or a long long must be tested whole, or one whose low word happens to be @@ -577,18 +2394,35 @@ void emit_ph2_ir(ph2_ir_t *p) case OP_cast: a64_mov(d, n); return; - case OP_branch: - a64_cbnz(p->src0_is_pointer || p->size_bytes == 8, n, - p->then_bb->elf_offset); - a64_b(p->else_bb->elf_offset); + case OP_ternary: + emit(a64_csel_insn(true, d, n, m, p->src2)); return; + case OP_branch: { + bool wide = p->src0_is_pointer || p->size_bytes == 8; + if (p->src1 || p->src2 == 2) { + int target = + p->src2 == 2 ? p->else_bb->elf_offset : p->then_bb->elf_offset; + int disp = (target - elf_code->size) / 4; + a64_check_disp(disp, 19, "arm64 conditional branch out of range"); + if (p->src1) + emit(a64_b_cond_insn((p->src1 - 1) ^ (p->src2 == 2), disp)); + else + emit(a64_cbz_insn(wide, n, disp)); + } else + a64_cbnz(wide, n, p->then_bb->elf_offset); + if (!p->src2) + a64_b(p->else_bb->elf_offset); + return; + } case OP_jump: - a64_b(p->next_bb->elf_offset); + if (!p->src2) + a64_b(p->next_bb->elf_offset); return; case OP_call: { func_t *f = find_func(p->func_name); if (!f) fatal("arm64 call to unknown function"); + if (!f->bbs) { if (!dynlink) fatal("arm64 external call requires --dynlink"); @@ -598,10 +2432,10 @@ void emit_ph2_ir(ph2_ir_t *p) return; } case OP_load_data_address: - a64_mov_imm(d, p->src0 + elf_data_start); + a64_address(d, p->src0 + elf_data_start); return; case OP_load_rodata_address: - a64_mov_imm(d, p->src0 + elf_rodata_start); + a64_address(d, p->src0 + elf_rodata_start); return; case OP_address_of_func: { func_t *f = find_func(p->func_name); @@ -618,7 +2452,7 @@ void emit_ph2_ir(ph2_ir_t *p) target = dynamic_sections.elf_plt_start + f->plt_offset; } else target = elf_code_start + f->bbs->elf_offset; - a64_mov_imm(A64_IP0, target); + a64_address(A64_IP0, target); a64_mem(A64_STORE, 8, A64_IP0, n, 0); return; } @@ -628,16 +2462,31 @@ void emit_ph2_ir(ph2_ir_t *p) case OP_indirect: emit(a64_blr_insn(A64_IP0)); return; - case OP_return: - if (p->src0 >= 0 && n != 0) + case OP_return: { + if (p->src0 >= 0) a64_mov(0, n); - a64_mov_imm(A64_IP0, ALIGN_UP(p->src1, A64_STACK_ALIGN)); - a64_add(A64_SP, A64_SP, A64_IP0); - emit(a64_ldp_post_insn(22, A64_GP, A64_SP, 16)); - emit(a64_ldp_post_insn(20, 21, A64_SP, 16)); - emit(a64_ldp_post_insn(A64_FP, A64_LR, A64_SP, 16)); + if (p->src2 & A64_SHARED_EXIT) { + a64_b(p->dest); + return; + } + int locals = ALIGN_UP(p->src1, A64_STACK_ALIGN), pop = 16; + if (p->src2 & A64_FOLDED_FRAME) + pop += locals; + else + a64_adjust_frame(locals, false); + for (int reg = + (((p->src2 & 15) + !!(p->src2 & A64_SAVE_GP) + 1) & ~1) - 2; + reg >= 0; reg -= 2) { + emit(a64_ldp_post_insn(a64_saved_reg(p->src2, reg), + a64_saved_reg(p->src2, reg + 1), A64_SP, + pop)); + pop = 16; + } + if (!(p->src2 & A64_FRAMELESS)) + emit(a64_ldp_post_insn(A64_FP, A64_LR, A64_SP, pop)); emit(a64_ret_insn()); return; + } default: fatal("unknown arm64 opcode"); } @@ -698,94 +2547,139 @@ void plt_generate(void) } } -void code_generate(void) +/* The process entry, up to the global initializers that follow it. The + * synthetic global frame is carved out of the runtime stack, and x19 keeps its + * base. + * + * A static image leaves argc and argv where the kernel put them, just above the + * frame. It is the first thing to run, so the stack below the entry SP is + * untouched anonymous memory and already reads as zero. + * + * A dynamic one parks them above the frame instead, since global initializers + * may use every allocated register, together with the original SP that + * __libc_start_main wants as stack_end. It also parks the base in the data + * segment, where main's prologue, entered from glibc, reads it back. And it + * clears the frame: the loader and glibc's startup have run over that memory + * first, so a global with no initializer would otherwise begin life holding + * their leftovers. + */ +static void a64_emit_entry(void) { int global_frame = ALIGN_UP(GLOBAL_FUNC->stack_size, A64_STACK_ALIGN); - /* At Linux process entry argc is at [sp] and argv begins at sp + 8; they - * are loaded into x20 and x21. Those two are allocatable by the global - * initialiser, so the values are parked in x23/x24 -- callee-saved, and - * outside this backend's allocation set -- to survive it. - */ - if (dynlink) - a64_mov_sp(25); + if (!dynlink) { + a64_adjust_frame(global_frame, true); + a64_mov_sp(A64_GP); + return; + } + a64_mov_sp(25); a64_mem(A64_LOAD, 8, 20, A64_SP, 0); emit(a64_add_imm_insn(true, 21, A64_SP, 8)); - a64_mov(23, 20); - a64_mov(24, 21); - if (dynlink) { - a64_mov_imm(A64_IP0, A64_STACK_ALIGN); - a64_sub(A64_SP, A64_SP, A64_IP0); - a64_mem(A64_STORE, 8, 23, A64_SP, 0); - a64_mem(A64_STORE, 8, 24, A64_SP, 8); - } - - /* The synthetic global frame is carved out of the runtime stack, and x19 - * keeps its base; only that base is parked in the data segment, where a - * prologue entered from glibc can read it back. - * - * Clearing it is the dynamic build's job alone. A static image is the first - * thing to run, so the stack below the entry SP is untouched anonymous - * memory and already reads as zero. A dynamic one has had the loader and - * glibc's startup run over that same memory first, so a global with no - * initializer would otherwise begin life holding their leftovers. - */ + a64_mov_imm(A64_IP0, 32); + a64_sub(A64_SP, A64_SP, A64_IP0); + a64_mem(A64_STORE, 8, 20, A64_SP, 0); + a64_mem(A64_STORE, 8, 21, A64_SP, 8); + a64_mem(A64_STORE, 8, 25, A64_SP, 16); a64_mov_imm(A64_IP0, global_frame); a64_sub(A64_SP, A64_SP, A64_IP0); a64_mov_sp(A64_GP); a64_mov_imm(A64_IP0, elf_data_start); a64_mem(A64_STORE, 8, A64_GP, A64_IP0, 0); - if (dynlink) { - /* Clear the frame inline rather than through libc's memset, which a - * --no-libc translation unit never declares. The frame is a multiple of - * sixteen bytes and may be empty: count x2 down to zero while x0 walks - * up through it. - */ - a64_mov(0, A64_GP); - a64_mov_imm(2, global_frame); - emit(a64_cbz_insn(true, 2, 4)); - emit(a64_mem_post_insn(A64_STORE, 8, A64_ZR, 0, 8)); - emit(a64_sub_imm_insn(true, 2, 2, 8)); - emit(a64_cbnz_insn(true, 2, -2)); - } - a64_b(GLOBAL_FUNC->bbs->elf_offset); - /* __syscall(number,arg1,...): AArch64 Linux wants x8,x0..x5. */ + + /* Clear the frame inline rather than through libc's memset, which a + * --no-libc translation unit never declares. The frame is a multiple of + * sixteen bytes and may be empty: count x2 down to zero while x0 walks up + * through it. + */ + a64_mov(0, A64_GP); + a64_mov_imm(2, global_frame); + emit(a64_cbz_insn(true, 2, 4)); + emit(a64_mem_post_insn(A64_STORE, 8, A64_ZR, 0, 8)); + emit(a64_sub_imm_insn(true, 2, 2, 8)); + emit(a64_cbnz_insn(true, 2, -2)); +} + +/* After the global initializers: call main with argc and argv, and leave with + * its result, through __libc_start_main when linking dynamically. + */ +static void a64_emit_main_call(void) +{ + int global_frame = ALIGN_UP(GLOBAL_FUNC->stack_size, A64_STACK_ALIGN); + if (!dynlink) { - a64_mov(8, 0); - a64_mov(0, 1); - a64_mov(1, 2); - a64_mov(2, 3); - a64_mov(3, 4); - a64_mov(4, 5); - a64_mov(5, 6); + /* Pass argc and argv only as far as main declares them. */ + int params = find_func("main")->num_params; + if (params > 0) + a64_mem(A64_LOAD, 8, 0, A64_SP, global_frame); + if (params > 1) + a64_add_sp(1, global_frame + 8); + emit(a64_bl_insn((MAIN_BB->elf_offset - elf_code->size) / 4)); + emit_arm64_wide_const(8, 93); emit(a64_svc_insn(0)); - emit(a64_ret_insn()); + return; } - for (ph2_ir_t *p = GLOBAL_FUNC->bbs->ph2_ir_list.head; p; p = p->next) - emit_ph2_ir(p); - if (MAIN_BB) { - if (dynlink) { - a64_mem(A64_LOAD, 8, 23, A64_SP, global_frame); - a64_mem(A64_LOAD, 8, 24, A64_SP, global_frame + 8); - a64_mov_imm(0, elf_code_start + MAIN_BB->elf_offset); - a64_mov(1, 23); - a64_mov(2, 24); - a64_mov(3, A64_ZR); - a64_mov(4, A64_ZR); - a64_mov(5, A64_ZR); - a64_mov(6, 25); - a64_bl_addr(dynamic_sections.elf_plt_start + PLT_FIXUP_SIZE); - emit(a64_brk_insn(0)); /* __libc_start_main does not return */ - } else { - a64_mov(0, 23); - a64_mov(1, 24); - emit(a64_bl_insn((MAIN_BB->elf_offset - elf_code->size) / 4)); - a64_mov_imm(8, 93); - emit(a64_svc_insn(0)); + a64_mem(A64_LOAD, 8, 23, A64_SP, global_frame); + a64_mem(A64_LOAD, 8, 24, A64_SP, global_frame + 8); + a64_mem(A64_LOAD, 8, 25, A64_SP, global_frame + 16); + a64_mov_imm(0, elf_code_start + MAIN_BB->elf_offset); + a64_mov(1, 23); + a64_mov(2, 24); + a64_mov(3, A64_ZR); + a64_mov(4, A64_ZR); + a64_mov(5, A64_ZR); + a64_mov(6, 25); + a64_bl_addr(dynamic_sections.elf_plt_start + PLT_FIXUP_SIZE); + emit(a64_brk_insn(0)); /* __libc_start_main does not return */ +} + +/* __syscall(number,arg1,...): AArch64 Linux wants x8,x0..x5. */ +static void a64_emit_syscall(void) +{ + a64_mov(8, 0); + a64_mov(0, 1); + a64_mov(1, 2); + a64_mov(2, 3); + a64_mov(3, 4); + a64_mov(4, 5); + a64_mov(5, 6); + emit(a64_svc_insn(0)); + emit(a64_ret_insn()); +} + +/* Pad with NOPs up to @offset, where a64_plan_offsets() placed a block. */ +static void a64_pad_to(int offset) +{ + while (elf_code->size < offset) + emit(a64_nop_insn()); +} + +void code_generate(void) +{ + a64_emit_entry(); + mop_visit_globals(emit_mop); + if (MAIN_BB) + a64_emit_main_call(); + if (a64_needs_syscall()) + a64_emit_syscall(); + + /* Walk the functions as a64_plan_offsets() did, padding up to the block + * offsets it planned. + */ + int index = 0; + for (func_t *f = FUNC_LIST.head; f; f = f->next) { + if (!f->bbs) + continue; + emit_mop(MOP_FLAT[index++]); + for (basic_block_t *bb = f->bbs; bb; bb = bb->rpo_next) { + a64_pad_to(bb->elf_offset); + for (vir_mop_t *mop = bb->mops.head; mop; mop = mop->next) { + index++; + emit_mop(mop); + } } } - for (int i = 0; i < ph2_ir_idx; i++) - emit_ph2_ir(PH2_IR_FLATTEN[i]); + if (index != mop_flat_count) + fatal("arm64 emission order differs from the flat order"); if (elf_code->size != elf_offset) fatal("arm64 code-size accounting mismatch"); if (dynlink) { diff --git a/src/arm64.c b/src/arm64.c index da61e83b..00a862df 100644 --- a/src/arm64.c +++ b/src/arm64.c @@ -89,21 +89,31 @@ int a64_sub_imm_insn(bool sf, int rd, int rn, int imm12) return a64_sf(sf) | 0x51000000 | ((imm12 & 0xfff) << 10) | (rn << 5) | rd; } +/* ORed into ADD or SUB #imm12, shifts the immediate left by 12 bits. */ +#define A64_IMM_LSL12 (1 << 22) + /* CMP #imm is SUBS into the zero register. */ int a64_cmp_imm_insn(bool sf, int rn, int imm12) { return a64_sf(sf) | 0x7100001f | ((imm12 & 0xfff) << 10) | (rn << 5); } -int a64_add_reg_insn(bool sf, int rd, int rn, int rm) +/* CMN #imm is ADDS into the zero register: for a nonzero @imm12 it sets every + * flag exactly as CMP #-imm12 would. + */ +int a64_cmn_imm_insn(bool sf, int rn, int imm12) { - return a64_sf(sf) | 0x0b000000 | (rm << 16) | (rn << 5) | rd; + return a64_sf(sf) | 0x3100001f | ((imm12 & 0xfff) << 10) | (rn << 5); } -int a64_sub_reg_insn(bool sf, int rd, int rn, int rm) -{ - return a64_sf(sf) | 0x4b000000 | (rm << 16) | (rn << 5) | rd; -} +/* Three-register data processing shares its operand bit layout. */ +#define A64_REGISTER_INSN(name, opcode) \ + int name(bool sf, int rd, int rn, int rm) \ + { \ + return a64_sf(sf) | opcode | (rm << 16) | (rn << 5) | rd; \ + } +A64_REGISTER_INSN(a64_add_reg_insn, 0x0b000000) +A64_REGISTER_INSN(a64_sub_reg_insn, 0x4b000000) /* CMP Rn, Rm is SUBS into the zero register. */ int a64_cmp_reg_insn(bool sf, int rn, int rm) @@ -137,57 +147,28 @@ int a64_mvn_insn(bool sf, int rd, int rm) { return a64_sf(sf) | 0x2a2003e0 | (rm << 16) | rd; } - -int a64_and_reg_insn(bool sf, int rd, int rn, int rm) -{ - return a64_sf(sf) | 0x0a000000 | (rm << 16) | (rn << 5) | rd; -} - -int a64_orr_reg_insn(bool sf, int rd, int rn, int rm) -{ - return a64_sf(sf) | 0x2a000000 | (rm << 16) | (rn << 5) | rd; -} - -int a64_eor_reg_insn(bool sf, int rd, int rn, int rm) -{ - return a64_sf(sf) | 0x4a000000 | (rm << 16) | (rn << 5) | rd; -} +A64_REGISTER_INSN(a64_and_reg_insn, 0x0a000000) +A64_REGISTER_INSN(a64_orr_reg_insn, 0x2a000000) +A64_REGISTER_INSN(a64_eor_reg_insn, 0x4a000000) /* MUL is MADD with the zero register as addend; MSUB computes Ra - Rn * Rm. */ -int a64_mul_insn(bool sf, int rd, int rn, int rm) -{ - return a64_sf(sf) | 0x1b007c00 | (rm << 16) | (rn << 5) | rd; -} +A64_REGISTER_INSN(a64_mul_insn, 0x1b007c00) int a64_msub_insn(bool sf, int rd, int rn, int rm, int ra) { return a64_sf(sf) | 0x1b008000 | (rm << 16) | (ra << 10) | (rn << 5) | rd; } -int a64_udiv_insn(bool sf, int rd, int rn, int rm) +/* MADD computes Ra + Rn * Rm. */ +int a64_madd_insn(bool sf, int rd, int rn, int rm, int ra) { - return a64_sf(sf) | 0x1ac00800 | (rm << 16) | (rn << 5) | rd; -} - -int a64_sdiv_insn(bool sf, int rd, int rn, int rm) -{ - return a64_sf(sf) | 0x1ac00c00 | (rm << 16) | (rn << 5) | rd; -} - -int a64_lslv_insn(bool sf, int rd, int rn, int rm) -{ - return a64_sf(sf) | 0x1ac02000 | (rm << 16) | (rn << 5) | rd; -} - -int a64_lsrv_insn(bool sf, int rd, int rn, int rm) -{ - return a64_sf(sf) | 0x1ac02400 | (rm << 16) | (rn << 5) | rd; -} - -int a64_asrv_insn(bool sf, int rd, int rn, int rm) -{ - return a64_sf(sf) | 0x1ac02800 | (rm << 16) | (rn << 5) | rd; + return a64_sf(sf) | 0x1b000000 | (rm << 16) | (ra << 10) | (rn << 5) | rd; } +A64_REGISTER_INSN(a64_udiv_insn, 0x1ac00800) +A64_REGISTER_INSN(a64_sdiv_insn, 0x1ac00c00) +A64_REGISTER_INSN(a64_lslv_insn, 0x1ac02000) +A64_REGISTER_INSN(a64_lsrv_insn, 0x1ac02400) +A64_REGISTER_INSN(a64_asrv_insn, 0x1ac02800) /* SXTB/SXTH/SXTW and UXTB/UXTH/UXTW into an X register are SBFM and UBFM taking * the low @bits bits. @@ -237,6 +218,19 @@ int a64_mem_op(int access, int size) return 0x38000000 | (sz << 30) | (1 << 22); } +/* LDR/STR with an X or zero-extended W offset, optionally scaled by size. */ +int a64_mem_register_insn(int access, + int size, + int rt, + int rn, + int rm, + bool scaled, + bool word_index) +{ + return a64_mem_op(access, size) | 0x204800 | (!word_index << 13) | + (rm << 16) | (scaled << 12) | (rn << 5) | rt; +} + /* LDUR/STUR: a signed 9-bit byte offset. */ int a64_mem_unscaled_insn(int access, int size, int rt, int rn, int ofs) { @@ -257,19 +251,6 @@ int a64_mem_post_insn(int access, int size, int rt, int rn, int ofs) (rn << 5) | rt; } -/* How many instructions a64_mem() in the code generator spends on an access of - * @size bytes at @ofs: one when either immediate form reaches it, and otherwise - * MOVZ/MOVK, ADD and the access through the scratch register. - */ -int a64_mem_count(int size, int ofs) -{ - if (ofs >= 0 && ofs <= 4095 * size && !(ofs % size)) - return 1; - if (ofs >= -256 && ofs <= 255) - return 1; - return 4; -} - /* STP pre-indexed and LDP post-indexed on X registers. The 7-bit offset is in * units of eight bytes. */ @@ -296,6 +277,11 @@ int a64_bl_insn(int disp) return 0x94000000 | (disp & 0x3ffffff); } +int a64_b_cond_insn(a64_cond_t cond, int disp) +{ + return 0x54000000 | ((disp & 0x7ffff) << 5) | cond; +} + int a64_cbz_insn(bool sf, int rt, int disp) { return a64_sf(sf) | 0x34000000 | ((disp & 0x7ffff) << 5) | rt; @@ -331,6 +317,13 @@ int a64_brk_insn(int imm16) return 0xd4200000 | ((imm16 & 0xffff) << 5); } +/* ADR: the address @disp bytes from the pc, which must lie within 1 MiB. */ +int a64_adr_insn(int rd, int disp) +{ + return 0x10000000 | ((disp & 3) << 29) | (((disp >> 2) & 0x7ffff) << 5) | + rd; +} + /* ADRP: the page of the target, @pages 4 KiB pages from the page of the pc. */ int a64_adrp_pages_insn(int rd, int pages) { @@ -338,6 +331,67 @@ int a64_adrp_pages_insn(int rd, int pages) rd; } +#undef A64_REGISTER_INSN + +int a64_lsl_imm_insn(bool sf, int rd, int rn, int shift) +{ + int bits = sf ? 63 : 31; + return (sf ? 0xd3400000U : 0x53000000U) | (((-shift) & bits) << 16) | + ((bits - shift) << 10) | (rn << 5) | rd; +} + +/* LSR or ASR by @shift: UBFM or SBFM taking bits @shift up to the top. */ +int a64_shr_imm_insn(bool sf, bool logical, int rd, int rn, int shift) +{ + int bits = sf ? 63 : 31; + return (logical ? (sf ? 0xd3400000U : 0x53000000U) + : (sf ? 0x93400000U : 0x13000000U)) | + (shift << 16) | (bits << 10) | (rn << 5) | rd; +} + +int a64_csel_insn(bool sf, int rd, int rn, int rm, int cond) +{ + return a64_sf(sf) | 0x1a800000 | (rm << 16) | (cond << 12) | (rn << 5) | rd; +} + +/* AND, ORR or EOR with the bitmask immediate whose N:immr:imms is @field. */ +int a64_logic_imm_insn(int opcode, bool sf, int rd, int rn, int field) +{ + return a64_sf(sf) | opcode | (field << 10) | (rn << 5) | rd; +} + +/* N:immr:imms for @value as a bitmask immediate of the X or W width, or -1: a + * pattern of 2, 4 up to 64 bits repeated, each a rotated run of ones. + */ +int a64_bitmask_field(unsigned long long value, bool sf) +{ + int size = sf ? 64 : 32; + unsigned long long element = sf ? value : value & 0xffffffffULL; + while (size > 2) { + int half = size / 2; + unsigned long long low = (1ULL << half) - 1; + if ((element & low) != ((element >> half) & low)) + break; + size = half; + element &= low; + } + unsigned long long all = size == 64 ? ~0ULL : (1ULL << size) - 1; + int ones = 0; + for (int bit = 0; bit < size; bit++) + ones += (element >> bit) & 1; + if (!ones || ones == size) + return -1; + unsigned long long run = (1ULL << ones) - 1; + for (int rotate = 0; rotate < size; rotate++) { + unsigned long long rotated = + rotate ? ((run >> rotate) | (run << (size - rotate))) & all : run; + if (rotated == element) + return ((size == 64) << 12) | (rotate << 6) | + (~(2 * size - 1) & 0x3f) | (ones - 1); + } + return -1; +} + int a64_nop_insn() { return 0xd503201f; diff --git a/src/defs.h b/src/defs.h index a3be7731..9bd02030 100644 --- a/src/defs.h +++ b/src/defs.h @@ -10,9 +10,6 @@ /* definitions */ -/* Common macro functions */ -#define is_newline(c) (c == '\r' || c == '\n') - /* Whether the C library in the output buffers its own stream I/O, so that a * whole block can be handed to fread() or fwrite() in one call. * @@ -43,14 +40,6 @@ #define MAX_STRING_LEN 4096 #define MAX_INCLUDE_DIRS 16 #define MAX_VAR_LEN 128 -/* ".label." plus an int, for basic_block_t's dump name. */ -#define MAX_LABEL_LEN 24 - -/* Staging buffer for one instruction's Graphviz label in bb_dump(). The widest - * case is a binary operator: three MAX_VAR_LEN names, three subscripts, an - * operator and 41 bytes of markup. - */ -#define DUMP_INSN_LEN 512 /* A type name is a struct, union, enum or typedef identifier, which the lexer * already bounds by MAX_ID_LEN; a narrower buffer rejects valid tags. @@ -67,7 +56,6 @@ #ifndef MAX_PARAMS #define MAX_PARAMS 8 #endif -#define MAX_LOCALS 3200 /* var_t itself is parsed as a record by every bootstrap stage. Leave room for * compiler metadata as well as user records with many declarators; C99 5.2.4.1 @@ -75,12 +63,11 @@ * this does not inflate scalar type storage. */ #define MAX_FIELDS 128 -#define MAX_TYPES 256 +/* The self-hosting compiler has outgrown the original 256-entry table. */ +#define MAX_TYPES 512 #define MAX_LABELS 256 /* Elements captured from an implicitly sized array initializer. */ #define MAX_IMPLICIT_ARRAY 256 -/* A self-compile emits ~101k ph2_ir; one pointer per slot in PH2_IR_FLATTEN. */ -#define MAX_IR_INSTR 262144 #define MAX_BB_PRED 128 #define MAX_BB_DOM_SUCC 64 #define MAX_CODE 262144 @@ -99,44 +86,34 @@ #define MAX_RELAPLT 1024 #define MAX_PLT 1024 #define MAX_GOTPLT 1024 -#define MAX_CONSTANTS 1024 #define MAX_NESTING 128 -/* How many instructions an if may speculate when flattened into a select, and - * how many blocks one of its arms may span. Beyond a handful, running the arm - * that would have been skipped costs more than the misprediction it avoids. - */ -#define MAX_SPECULATED_INSNS 8 -#define MAX_IF_ARM_BLOCKS 4 - -/* Recursion limits for nesting in the input. The parser descends recursively - * for each of these, so a deeply nested program would otherwise exhaust the - * machine stack before any diagnostic could be produced. +/* Recursion limits for nesting in the input. The parser and the preprocessor + * descend recursively for each of these, so a deeply nested program would + * otherwise exhaust the machine stack before any diagnostic could be produced. */ #define MAX_EXPR_DEPTH 256 #define MAX_BLOCK_DEPTH 256 +#define MAX_MACRO_DEPTH 256 #define MAX_OPERAND_STACK_SIZE 32 /* Depth of the operator stack that read_expr() and the constant-expression * evaluator shunt through, and of the value stack the latter keeps beside it. */ #define MAX_OPERATOR_STACK_SIZE 10 -#define MAX_ANALYSIS_STACK_SIZE 1600 /* Default capacities for common data structures, with the arena sizes taken * from typical usage patterns. */ #define DEFAULT_ARENA_SIZE 262144 /* 256 KiB - standard default */ #define SMALL_ARENA_SIZE 65536 /* 64 KiB - for small allocations */ -#define LARGE_ARENA_SIZE 524288 /* 512 KiB - for instruction arena */ +#define LARGE_ARENA_SIZE 524288 /* 512 KiB - for token storage */ #define DEFAULT_FUNCS_SIZE 64 #define DEFAULT_SRC_FILE_COUNT 8 /* Arena compaction bitmask flags for selective memory reclamation */ #define COMPACT_ARENA_BLOCK 0x01 /* BLOCK_ARENA - variables/blocks */ -#define COMPACT_ARENA_INSN 0x02 /* INSN_ARENA - instructions */ #define COMPACT_ARENA_BB 0x04 /* BB_ARENA - basic blocks */ -#define COMPACT_ARENA_HASHMAP 0x08 /* HASHMAP_ARENA - hash nodes */ #define COMPACT_ARENA_GENERAL 0x10 /* GENERAL_ARENA - misc allocations */ #define ELF_START 0x10000 @@ -159,12 +136,6 @@ ; \ } while (0) -/* shecc runs on the target it compiles for, so the host pointer width is the - * target's. This must not be hardcoded to 4: on an LP64 target the var_list - * allocations and memcpy sizes below would be half what they need. - */ -#define HOST_PTR_SIZE PTR_SIZE - /* shecc parses no attributes, and does not need the hint: it reports what it * cannot compile rather than warning about it. */ @@ -172,8 +143,6 @@ #else /* suppress GCC/Clang warnings */ #define UNUSED(x) (void) (x) -/* configure host data model when using 'memcpy'. */ -#define HOST_PTR_SIZE __SIZEOF_POINTER__ /* Marks the diagnostic paths that never come back. Without it the host compiler * cannot see that a variable set on every surviving path is initialized, and @@ -209,49 +178,17 @@ #define ELF_IS_64 0 #endif -/* Ceilings for strength_reduce(): how long a chain of instructions may compute - * one address, and how many rounds the step analysis takes to settle. - */ -#define MAX_IV_CHAIN 12 -#define MAX_IV_ROUNDS 4 - -/* A pointer the loop advances costs a register for the whole loop and an - * addition at the bottom of it, so it only pays where it replaces more work - * than that -- and only a couple of them fit before the loop's own variables - * start going to the frame instead. - */ -#define MIN_IV_CHAIN 3 -#define MAX_IV_PER_LOOP 2 - -/* How many blocks one natural loop's walk keeps in hand at once. Past this the - * walk stops widening, which can only understate a depth. - */ -#define MAX_LOOP_WALK 512 - -/* Limits on copying a function into its callers: how big a body is worth - * copying, how many distinct variables one such body may name, and how many - * times the pass sweeps the program so that a function which becomes copyable - * only after its own callee was copied into it still gets copied. - */ -#define MAX_INLINE_INSNS 16 -#define MAX_INLINE_VARS 32 -#define MAX_INLINE_ROUNDS 3 - -/* What a naming inside one loop is worth against one in straight-line code, and - * how many nesting levels still multiply it. +/* What a use inside one loop is worth against one in straight-line code when + * ranking values for a register, and how many nesting levels still multiply it. */ #define LOOP_USE_WEIGHT 8 -#define MAX_WEIGHTED_LOOP_DEPTH 3 +#define MAX_WEIGHTED_LOOP_DEPTH 4 /* How many registers at the top of the allocator's file a call preserves. Only * such a register can hold a value across a call, so only these may be given to * a variable for the whole of a function that calls anything. A target states * its own count in mk/.mk, beside the REG_CNT that fixes the file it * counts from; a target that has not had its file checked this way keeps none. - * - * HAVE_COND_MOVE comes from the same place and says whether the target can - * select between two values without branching, which is what makes flattening - * an if into a select worthwhile. */ #ifndef CALLEE_SAVED_REGS #define CALLEE_SAVED_REGS 0 @@ -268,27 +205,49 @@ typedef struct arena_block { typedef struct { arena_block_t *head; int total_bytes; /* Track total allocation for profiling */ + int peak_bytes; /* High-water payload capacity before compaction. */ int block_size; /* Default block size for new blocks */ } arena_t; +typedef struct { + arena_block_t *head; + int offset; +} arena_mark_t; + +arena_mark_t arena_mark(arena_t *arena); +int arena_bytes_since(arena_t *arena, arena_mark_t mark); +/* Discard allocations after a live mark; return false if the mark is stale. */ +bool arena_rewind(arena_t *arena, arena_mark_t mark); + /* string-based hash map definitions */ typedef struct hashmap_node { char *key; void *val; + int hash; /* hashmap_hash() of key, so a probe compares strings rarely */ bool occupied; + bool owns_key; } hashmap_node_t; typedef struct { int size; int cap; hashmap_node_t *table; + bool has_owned_keys; } hashmap_t; +hashmap_t *hashmap_create(int cap); +void hashmap_put_borrowed(hashmap_t *map, char *key, void *val); +hashmap_node_t *hashmap_get_node(hashmap_t *map, char *key); +void *hashmap_get(hashmap_t *map, char *key); +bool hashmap_contains(hashmap_t *map, char *key); +void hashmap_free(hashmap_t *map); + /* lexer tokens */ typedef enum { T_start, /* FIXME: Unused, intended for lexer state machine init */ - T_eof, /* end-of-file (EOF) */ + T_translation_unit, /* internal boundary between source input files */ + T_eof, /* end-of-file (EOF) */ T_numeric, T_floating, /* C99 floating literal; lowering is staged separately */ T_identifier, @@ -430,6 +389,9 @@ typedef struct token_stream { token_t *tail; } token_stream_t; +bool numeric_suffix_is_valid(const char *suffix); +int numeric_literal_base(const char *literal); + /* String pool for identifier deduplication */ typedef struct { hashmap_t *strings; /* Map string -> interned string */ @@ -461,11 +423,6 @@ typedef enum { /* intermediate use in front-end. No code generation */ OP_generic, - OP_phi, - OP_unwound_phi, /* work like address_of + store */ - /* rd = rs2 ? rs1 : rs3 -- select without a branch. */ - OP_cmov, - /* calling convention */ OP_define, /* function entry point */ OP_push, /* prepare arguments */ @@ -473,7 +430,8 @@ typedef enum { OP_indirect, /* indirect call with function pointer */ OP_return, /* explicit return */ - OP_allocat, /* allocate space on stack */ + OP_va_start, /* address of the first unnamed ABI argument */ + OP_allocat, /* allocate space on stack */ OP_assign, OP_load_constant, /* load constant */ OP_load_data_address, /* lookup address of a constant in data section */ @@ -483,12 +441,10 @@ typedef enum { OP_branch, /* conditional jump */ OP_jump, /* unconditional jump */ OP_func_ret, /* returned value */ - OP_label, /* for goto label */ /* function pointer */ OP_address_of_func, /* resolve function entry */ OP_load_func, /* prepare indirective call */ - OP_global_load_func, /* memory address operations */ OP_address_of, /* lookup variable's address */ @@ -527,45 +483,20 @@ typedef enum { /* data type conversion */ OP_trunc, OP_sign_ext, - OP_cast, - - /* entry point of the state machine */ - OP_start + OP_cast } opcode_t; -/* variable definition */ - -/* Depth of the SSA renaming stack: how many definitions of one variable can be - * live along a single dominator path. - */ -#define MAX_RENAME_STACK 64 - -typedef struct { - int counter; - - /* Grown on demand: only a base variable is ever renamed, so the SSA - * versions copied from it -- the large majority of all variables -- would - * otherwise each carry an unused MAX_RENAME_STACK array. - */ - int *stack; - int stack_idx; - int stack_cap; -} rename_t; - -typedef struct ref_block ref_block_t; - -struct ref_block_list { - ref_block_t *head, *tail; -}; - -typedef struct ref_block_list ref_block_list_t; +#define OP_USUAL_ARITHMETIC_CASES \ + case OP_add: \ + case OP_sub: \ + case OP_mul: \ + case OP_div: \ + case OP_mod: \ + case OP_bit_and: \ + case OP_bit_or: \ + case OP_bit_xor -typedef struct insn insn_t; - -typedef struct use_chain_node { - insn_t *insn; - struct use_chain_node *next, *prev; -} use_chain_t; +/* variable definition */ typedef struct var var_t; typedef struct type type_t; @@ -578,6 +509,7 @@ typedef struct var_list { struct var { type_t *type; + /* Lexical owner, used while parsing declarator constant expressions. */ void *scope; @@ -589,25 +521,85 @@ struct var { * intern_string(). Never NULL -- an unnamed variable holds "". */ char *var_name; - int ptr_level; - bool is_func; + + /* Original C spelling when an internal-linkage name is mangled for the + * merged object/function tables. NULL for names emitted as-is. + */ + char *source_name; /* A pointer-to-callback object is not callable itself. Its compact callback * pointee signature is restored only after one dereference. */ void *pointee_func_signature; + struct var *compound_literal_address; - /* Set only for the unparenthesized `result name(parameters)` spelling. - * Block typedef support uses it to keep direct function aliases distinct - * from the older parenthesized pointer-to-function declarator path. + /* A non-NULL field means the reference is a bit-field and must use the + * mask-and-merge store path rather than a byte/word OP_write. */ - bool is_direct_function_declarator; + struct var *compound_literal_bitfield; + + /* A function-pointer declarator owns a prototype separately from its value + * type. Keeping this syntax-only object lets an indirect call use the same + * argument lowering as a direct call (notably record-by-value arguments) + * without putting function ABI details into type_t. Kept void-typed so the + * self-hosted parser does not need an incomplete `struct func` declaration + * while reading var_t itself. parser.c owns the cast back to func_t. + */ + void *func_signature; + int ptr_level; /* The parenthesized callback spelling `int (*name)(int)` is distinct from a * direct function declarator. Block typedef lowering uses this count to * admit exactly one pointer layer as a callback-pointer alias. */ int parenthesized_function_pointer_level; + + /* One bit per pointer level, counted from the base type. The legacy + * is_const_pointer flag describes only the outermost level; this retains + * qualifiers on intermediate pointers such as `int * const *`. + */ + unsigned int pointer_const_mask; + unsigned int pointer_volatile_mask; + int array_size; + int array_dim2; /* second dimension size for multidimensional arrays */ + int array_dim3; /* third dimension size for multidimensional arrays */ + int array_dim4; /* fourth dimension size for multidimensional arrays */ + + /* Bounds of the array addressed by a parenthesized pointer declarator, e.g. + * `int (*row)[2]`. They describe the pointee, not this pointer-sized + * object, so they must never participate in size_var(). + */ + int pointee_array_size; + int pointee_array_dim2; + int pointee_array_dim3; + int pointee_array_dim4; + + /* Pointer depth of one element in the array described above. This + * distinguishes `int (*p)[2]` from `int *(*p)[2]`: both address rows, but + * the latter's row elements are pointers. + */ + int pointee_array_element_ptr_level; + int offset; /* offset from stack or frame, index 0 is reserved */ + int vir_source_index; + int bit_width; + int bit_offset; + int bit_storage_size; + int init_val; /* for global initialization */ + int init_val_hi; /* upper word of an 8-byte integer constant */ + + /* For the address of a static object in a constant initializer, the bytes + * one integer step moves it by, so `1 + array` advances as `array + 1` + * does; 0 where no such step is known. + */ + int address_stride; + bool is_func; + bool is_function_designator; + + /* Set only for the unparenthesized `result name(parameters)` spelling. + * Block typedef support uses it to keep direct function aliases distinct + * from the older parenthesized pointer-to-function declarator path. + */ + bool is_direct_function_declarator; bool parenthesized_function_pointer_const; bool parenthesized_function_pointer_restrict; @@ -643,110 +635,23 @@ struct var { bool has_initializer; /* a file-scope definition supplied an initializer */ bool is_const_qualified; /* true if variable has const qualifier */ bool is_volatile; /* declaration used the volatile qualifier */ + bool is_volatile_access; /* computed address designates volatile storage */ bool is_const_pointer; /* true for the outermost `* const` qualifier */ - /* One bit per pointer level, counted from the base type. The legacy - * is_const_pointer flag describes only the outermost level; this retains - * qualifiers on intermediate pointers such as `int * const *`. - */ - unsigned int pointer_const_mask; - bool address_taken; /* true if variable address was taken (&var) */ - /* Working state for strength_reduce(): how many instructions in the - * function write the variable, whether it is written inside the loop being - * examined, and how much its value moves per iteration when it does. All - * three are recomputed per loop; nothing outside that pass reads them. - */ - int def_cnt; - int loop_stamp; - int iv_gen; - int iv_step; - - /* pin_registers()'s tally for the variable: what its namings are worth - * weighted by loop depth, the reverse-post-order number of the last block - * that named it, whether it was ever named in two, and whether a loop named - * it. Stamped per function so that no array has to hold the candidates -- - * the file has a handful of registers and a function names hundreds of - * variables, and the one worth a register is not reliably among the first - * few met. - */ - int pin_gen; - int pin_weight; - int pin_blk; - bool pin_cross; - bool pin_hot; - - /* Defined inside an arm that if_convert() flattened into a select. The - * register its variable is pinned to still holds the value flowing into the - * select, which the arms read and the select overwrites, so a value - * computed on the way there must go somewhere else. - */ - bool in_select_arm; - int array_size; + bool address_taken; /* true if variable address was taken (&var) */ bool has_direct_array_declarator; bool has_unsized_array; /* `T name[]`: bound is supplied by initializer */ + /* `T member[]` at the end of a struct has no initializer-supplied bound and * contributes no bytes to the record's fixed layout. */ bool is_flexible_array_member; - int array_dim2; /* second dimension size for multidimensional arrays */ - int array_dim3; /* third dimension size for multidimensional arrays */ - int array_dim4; /* fourth dimension size for multidimensional arrays */ - /* Bounds of the array addressed by a parenthesized pointer declarator, e.g. - * `int (*row)[2]`. They describe the pointee, not this pointer-sized - * object, so they must never participate in size_var(). - */ - int pointee_array_size; bool has_direct_pointee_array_declarator; - int pointee_array_dim2; - int pointee_array_dim3; - int pointee_array_dim4; - /* Pointer depth of one element in the array described above. This - * distinguishes `int (*p)[2]` from `int *(*p)[2]`: both address rows, but - * the latter's row elements are pointers. - */ - int pointee_array_element_ptr_level; - int offset; /* offset from stack or frame, index 0 is reserved */ /* Record bit-field metadata. `offset` is the containing storage unit's byte * offset. `is_bitfield` distinguishes an ordinary member from the valid * unnamed zero-width field used as an allocation-unit barrier. */ bool is_bitfield; - int bit_width; - int bit_offset; - int bit_storage_size; - int init_val; /* for global initialization */ - int init_val_hi; /* upper word of an 8-byte integer constant */ - /* Generation stamps used by compute_live_in() to test set membership in - * constant time instead of rescanning live_kill and live_in per element. - */ - int kill_gen; - int in_gen; - /* Stamp for the successor-union set merge_live_in() builds. */ - int merge_gen; - struct var *base; - int subscript; - - /* Every SSA version of this variable, grown on demand. A fixed 128-entry - * array made every var_t 1 KiB heavier -- and var_t is embedded by value in - * type_t's field table and in every function's parameter list -- while the - * append was unchecked, so a variable assigned more than 128 times in one - * function wrote past the end. - */ - struct var **subscripts; - int subscripts_idx; - int subscripts_cap; - - /* SSA renaming state, allocated on first use by var_rename(). Only a base - * variable is ever renamed, so the versions copied from it -- the large - * majority of all variables -- each carried an unused 24-byte rename_t - * inline. Field order here is load-bearing: reordering var_t breaks the - * bootstrap, so the pointer stays where the struct sat. - */ - rename_t *rename; - ref_block_list_t ref_block_list; /* blocks which kill variable */ - use_chain_t *users_head, *users_tail; - struct insn *last_assign; - int consumed; bool is_ternary_ret; bool is_logical_ret; bool is_const; /* whether a constant representaion or not */ @@ -757,22 +662,9 @@ struct var { * nothing uses the value. */ bool is_assignment_reload; - int phys_reg; /* low physical register (-1 if unassigned) */ - /* The high word of a 32-bit-target wide scalar. It remains -1 for the - * ordinary single-register representation and on LP64 targets. - */ - int phys_reg_hi; - int first_use; /* First instruction index where variable is used */ - int last_use; /* Last instruction index where variable is used */ - int use_count; /* Number of times variable is used */ bool space_is_allocated; /* whether space is allocated for this variable */ bool has_backing_storage; - /* This flag is used to indicate to the compiler that the offset of the - * variable is based on the top of the local stack. - */ - bool ofs_based_on_stack_top; - /* True when this variable was synthesized to hold a compound literal (e.g., * array or struct literal temporaries). */ @@ -785,7 +677,6 @@ struct var { * object rather than the temporary. */ bool is_compound_literal_reference; - struct var *compound_literal_address; /* A record value with no storage of its own: its bytes are those of the * object at compound_literal_address. A member selection or address-of uses @@ -793,23 +684,12 @@ struct var { */ bool defers_record_copy; - /* A non-NULL field means the reference is a bit-field and must use the - * mask-and-merge store path rather than a byte/word OP_write. - */ - struct var *compound_literal_bitfield; - /* String literals have immutable storage duration in C. Preserve that * provenance separately from the pointer type so the compatibility warning * can remain opt-in while legacy source still compiles. */ bool is_string_literal; - /* For the address of a static object in a constant initializer, the bytes - * one integer step moves it by, so `1 + array` advances as `array + 1` - * does; 0 where no such step is known. - */ - int address_stride; - /* The null pointer constant `(void *) 0` (C99 6.3.2.3p3), which unlike any * other void pointer converts to a function pointer. */ @@ -828,15 +708,6 @@ struct var { */ bool is_func_name_array_address; - /* A function-pointer declarator owns a prototype separately from its value - * type. Keeping this syntax-only object lets an indirect call use the same - * argument lowering as a direct call (notably record-by-value arguments) - * without putting function ABI details into type_t. Kept void-typed so the - * self-hosted parser does not need an incomplete `struct func` declaration - * while reading var_t itself. parser.c owns the cast back to func_t. - */ - void *func_signature; - /* C ABI lowering passes record parameters as pointers to caller-owned * copies. The source-level declaration remains a record so field access and * record assignment keep their C semantics; OP_address_of materializes the @@ -857,8 +728,9 @@ typedef struct typedef_binding { struct block { var_list_t locals; - /* C tags and enumeration constants have lexical, not translation-unit, - * scope. Variables remain in locals; these lists serve parser lookups. + /* Tags and enumerators follow C lexical scope. File-scope tag/enum bindings + * live on the active translation-unit root; typedef bindings remain on the + * migration path through TYPES. Variables remain in locals. */ void *type_tags; void *constants; @@ -874,6 +746,7 @@ int read_const_sizeof_type(block_t *scope); int read_const_wstring_size(void); int read_sizeof_constant(block_t *scope); int read_const_expr_operand(block_t *scope); +int alignment_type(type_t *type); void add_block_typedef(block_t *block, char name[], type_t *type); bool find_block_typedef(block_t *block, const char *name); type_t *find_visible_type(const char *name, block_t *block); @@ -890,83 +763,21 @@ typedef struct basic_block basic_block_t; * @size: Current number of elements in the array * @capacity: Number of elements that can be stored without resizing * @elements: Pointer to the array of characters - * @plain_source: set by the lexer once it has checked that a source buffer has - * no trigraph and no line splice, so phases 1 and 2 are the - * identity on it + * @plain_end: the lexer's mark of the next possible trigraph or line splice, + * so phases 1 and 2 are the identity from its read cursor, which + * it keeps in @size, up to it */ typedef struct { int size; int capacity; char *elements; - bool plain_source; + int plain_end; } strbuf_t; -/* phase-2 IR definition */ -struct ph2_ir { - /* Grouped by width so the struct carries no interior padding: mixed in - * declaration order it was 72 bytes for 63 bytes of fields, on all ~101k of - * them a self-compile emits. Callee / definition name, interned in - * GENERAL_ARENA rather than copied. A MAX_VAR_LEN array here was 128 of - * this struct's 192 bytes while only OP_define, OP_call and - * OP_address_of_func ever name anything. NULL when unused. - */ - char *func_name; - basic_block_t *next_bb; - basic_block_t *then_bb; - basic_block_t *else_bb; - struct ph2_ir *next; - - opcode_t op; - int src0; - int src1; - /* The register OP_cmov keeps when its condition does not hold. */ - int src2; - int dest; - - /* A 32-bit target represents a wide integer as low/high register pairs. -1 - * means this instruction uses the existing single-register form. - */ - int src0_hi; - int src1_hi; - int dest_hi; - /* Type information for LP64 support */ - int size_bytes; /* Size in bytes for load/store/read/write operations */ - - bool is_branch_detached; - - /* When an instruction uses a variable that its offset is based on the top - * of the stack, this instruction's flag is also set to indicate the - * compiler to recalculate the offset after the function's stack size has - * been determined. - * - * Currently, only OP_load, OP_store and OP_address_of need this flag to - * recompute the offset. - */ - bool ofs_based_on_stack_top; - bool is_pointer; /* True if this operation involves a pointer type */ - /* Scalar signedness accompanies register values independently of their - * storage width. Comparisons inspect their sources; arithmetic and loads - * inspect the result. - */ - bool is_unsigned; - - /* Operand provenance is required by LP64 backends: pointer arithmetic keeps - * the address operand wide but sign-extends an int index. - */ - bool src0_is_pointer; - bool src1_is_pointer; - bool src0_is_unsigned; - bool src1_is_unsigned; - - /* The load, read or store accesses a volatile object, which is a side - * effect whether or not its value is used (C99 6.7.3p6), so no rewrite may - * drop it, replace a load with a copy of a value read earlier, or drop a - * store for writing what the object already holds. - */ - bool is_volatile; -}; - -typedef struct ph2_ir ph2_ir_t; +/* An allocated machine operation, defined with VIR's machine layer in + * vir-lower.h. + */ +typedef struct vir_mop vir_mop_t; /* type definition */ struct type { @@ -1063,6 +874,7 @@ struct type { * subsequently written at a use site. */ unsigned int pointer_const_mask; + unsigned int pointer_volatile_mask; bool is_union; /* preserves union semantics for anonymous typedef unions */ bool has_flexible_array_member; /* cannot be embedded by value in a record */ @@ -1143,47 +955,9 @@ typedef struct type_tag { struct type_tag *next; } type_tag_t; -struct phi_operand { - var_t *var; - basic_block_t *from; - struct phi_operand *next; -}; - -typedef struct phi_operand phi_operand_t; - -struct insn { - struct insn *next, *prev; - int idx; - opcode_t opcode; - var_t *rd; - var_t *rs1; - var_t *rs2; - - /* The value OP_cmov keeps when its condition does not hold. A select needs - * three inputs and the two source fields are taken by the chosen value and - * the condition; every other opcode leaves this NULL. - */ - var_t *rs3; - int sz; - bool useful; /* Used in DCE process. Set true if instruction is useful. */ - basic_block_t *belong_to; - phi_operand_t *phi_ops; - - /* Callee or goto-label name, interned rather than copied. add_insn() - * already interned the text before copying it in, so the array was 64 of - * this struct's 136 bytes for a string the pool owns anyway, on all ~73k - * instructions a self-compile builds. NULL when the opcode names nothing. - */ - char *str; -}; - -typedef struct { - insn_t *head, *tail; -} insn_list_t; - typedef struct { - ph2_ir_t *head, *tail; -} ph2_ir_list_t; + vir_mop_t *head, *tail; +} vir_mop_list_t; typedef enum { NEXT, ELSE, THEN } bb_connection_type_t; @@ -1192,31 +966,10 @@ typedef struct { bb_connection_type_t type; } bb_connection_t; -struct symbol { - var_t *var; - int index; - struct symbol *next; -}; - -typedef struct symbol symbol_t; - -typedef struct { - symbol_t *head, *tail; -} symbol_list_t; - struct basic_block { - /* Members are grouped by width -- 16-byte lists, then pointers, then ints, - * then the flags -- so the struct carries no interior padding. Mixed in - * declaration order it was 320 bytes for 302 bytes of fields, on every one - * of the ~51k blocks a self-compile creates. - */ - insn_list_t insn_list; - ph2_ir_list_t ph2_ir_list; - var_list_t live_gen; - var_list_t live_kill; - var_list_t live_in; - var_list_t live_out; - symbol_list_t symbol_list; /* variable declaration */ + struct vir_block *vir; + vir_mop_list_t mops; + unsigned int machine_use, machine_def, machine_liveout; /* Predecessor edges, grown on demand. A fixed MAX_BB_PRED array cost two * kilobytes in every basic block -- by far the largest thing in one -- @@ -1224,50 +977,10 @@ struct basic_block { */ bb_connection_t *prev; - /* Register file on entry to this block, captured by reg_alloc() at the end - * of the predecessor it falls out of. Non-NULL only when a file was handed - * over, and bb_export_regs() does that only for an edge that is all three - * of: the predecessor's single successor, that predecessor's rpo_next, and - * this block's single predecessor. A sole predecessor alone is NOT enough - * -- a branch target is emitted wherever the backend's linear walk puts it, - * so the registers reaching it are not the ones the branch left. - * - * Allocated only for a block that actually inherits a file: fewer than one - * block in thirteen does, so a REG_CNT array here cost 64 bytes on all ~51k - * blocks to serve 7% of them. - */ - struct var **entry_regs; - - /* Used in instruction dumping when ir_dump is enabled, and allocated only - * then: a fixed array here is 128 bytes on every one of the tens of - * thousands of blocks a self-compile creates, all of it zeroed for nothing - * in the default path. - */ - char *bb_label_name; struct basic_block *next; /* normal BB */ struct basic_block *then_; /* conditional BB */ struct basic_block *else_; - struct basic_block *idom; - struct basic_block *r_idom; struct basic_block *rpo_next; - struct basic_block *rpo_r_next; - - /* Dominance and reverse-dominance frontiers. These were fixed worst-case - * arrays sized MAX_BB_DOM_SUCC / MAX_BB_RDOM_SUCC, which cost 2560 bytes in - * every basic block while a typical block uses a handful of entries. Worse, - * the appends were unchecked and a self-compile really does push df_idx to - * 72, overrunning a 64-entry DF into the RDF that followed it. Growing them - * on demand removes both the waste and the fixed ceiling. - */ - struct basic_block **DF; - struct basic_block **RDF; - - /* Dominator-tree children, grown on demand for the same reason as prev[] - * and the frontiers: a fixed array cost half a kilobyte in every block. - */ - struct basic_block **dom_next; - struct basic_block *dom_prev; - struct basic_block *rdom_prev; func_t *belong_to; block_t *scope; @@ -1281,40 +994,17 @@ struct basic_block { */ int prev_idx; - /* Index of this block's first instruction in PH2_IR_FLATTEN, or -1 when it + /* Index of this block's first instruction in MOP_FLAT, or -1 when it * emitted none. Recorded while that mapping is built so the backend need * not search for it. */ - int ph2_base; + int mop_base; int rpo; - int rpo_r; - - /* How many loops enclose the block. pin_registers() weights a variable's - * uses by it: a name inside a loop stands for as many reads as the loop has - * iterations, and ranking by the plain count gave a register to a variable - * named four times in straight-line code over one named once in the - * innermost loop. - */ - int loop_depth; - - /* What one naming of a variable in this block is worth to pin_registers(), - * derived from loop_depth once rather than on every operand of every - * instruction the tally walks. - */ - int loop_weight; + int elf_offset; - /* Stamp marking the block as already counted for the loop being walked, so - * that one loop raises its depth once however many ways in there are. + /* VIR block-map index plus one; zero means this parser block is unmapped. */ - int loop_mark; - int df_idx; - int rdf_idx; - int df_cap; - int rdf_cap; - int visited; - int dom_next_idx; - int dom_next_cap; - int elf_offset; + int vir_map_id; /* Whether any emitted branch or jump names this block as its target. A * block no edge jumps to is reached only by falling out of the block @@ -1322,19 +1012,13 @@ struct basic_block { * registers hold across the boundary. */ bool is_branch_target; - bool useful; /* indicate whether this BB contains useful instructions */ -}; -struct ref_block { - basic_block_t *bb; - struct ref_block *next; + /* Whether a branch from later in the layout targets this block, so that it + * starts a loop body. ARM64 aligns such blocks. + */ + bool is_loop_start; }; -/* Syntactic representation of func, combines syntactic details (e.g., return - * type, parameters) with SSA-related information (e.g., basic blocks, control - * flow) to support parsing, analysis, optimization, and code generation. - */ - typedef struct { char label_name[MAX_ID_LEN]; basic_block_t *bb; @@ -1371,35 +1055,24 @@ struct func { */ bool is_inline; - /* inline_calls()'s verdict on this body and the return that ends it, - * stamped with the round that reached them: a body is examined once per - * round rather than once per call site that names it. - */ - int inline_gen; - bool inline_ok; - struct insn *inline_ret; int stack_size; + bool passes_stack_args; + + /* A test the ARM64 prologue runs ahead of the frame, returning at once when + * it holds; see a64_early_exit(). NULL when there is none. + */ + vir_mop_t *early_test; + int early_cond; + int early_value; + void *vir_context; - /* SSA info */ + /* Parser cursors become the allocated machine CFG after lowering. */ basic_block_t *bbs; basic_block_t *exit; - symbol_list_t global_sym_list; - int bb_cnt; - int visited; - /* How many callee-saved registers this function's prologue must preserve, - * counted from RBX upward. Functions that never need them pay nothing. - */ + /* Callee-saved registers the backend must preserve. */ int saved_regs; - /* Registers holding a variable for the whole function, one bit each. The - * backend needs these: its notion of what is live out of a block comes from - * the successor's entry registers, which say nothing about a variable that - * is resident everywhere, and it would otherwise drop the code that puts a - * value into one as dead. - */ - int pinned_regs; - /* Information used for dynamic linking */ bool is_used; /* A direct aggregate-return call requires shecc's private sret ABI. */ @@ -1413,17 +1086,22 @@ typedef struct { func_t *head, *tail; } func_list_t; -typedef struct { - func_t *func; - basic_block_t *bb; - void (*preorder_cb)(func_t *, basic_block_t *); - void (*postorder_cb)(func_t *, basic_block_t *); -} bb_traversal_args_t; - -typedef struct { - var_t *var; - int polluted; -} regfile_t; +/* Iterate over functions with bodies, skipping declarations. */ +#define FOR_EACH_FUNCTION_BODY(func) \ + for (func_t *func = FUNC_LIST.head; func; func = func->next) \ + if ((func)->bbs) + +/* Load a two-word value without clobbering its address register. */ +#define EMIT_PAIR_LOAD(load, rd, rd_hi, base) \ + do { \ + if ((rd) == (base)) { \ + load(rd_hi, base, 4); \ + load(rd, base, 0); \ + } else { \ + load(rd, base, 0); \ + load(rd_hi, base, 4); \ + } \ + } while (0) /* In the ELF specification, the following data types are defined for data * representation: @@ -1555,4 +1233,13 @@ typedef struct { int r_addend; /* Elf32_Sword */ } elf32_rela_t; +bool op_is_binary_alu(opcode_t op); +bool op_writes_dest(opcode_t op); +bool op_is_integer_binary(opcode_t op); +bool op_is_comparison(opcode_t op); +bool op_is_scalar_unary(opcode_t op); +bool op_is_scalar_cast(opcode_t op); +const char *opcode_mnemonic(opcode_t op); +basic_block_t *bb_sole_pred(const basic_block_t *bb); + #define ELF32_ST_INFO(b, t) (((b) << 4) + ((t) & 0xf)) diff --git a/src/elf.c b/src/elf.c index b6bb6855..297ece69 100644 --- a/src/elf.c +++ b/src/elf.c @@ -15,6 +15,14 @@ #define PAGESIZE 4096 #endif +static bool elf_dynamic_sections_ready(const strbuf_t *relplt) +{ + return !dynlink || + (dynamic_sections.elf_interp && relplt && dynamic_sections.elf_plt && + dynamic_sections.elf_got && dynamic_sections.elf_dynstr && + dynamic_sections.elf_dynsym && dynamic_sections.elf_dynamic); +} + int elf_symbol_index = 0; void elf_write_str(strbuf_t *elf_array, const char *vals) @@ -76,6 +84,13 @@ void elf_write_blk(strbuf_t *elf_array, const void *blk, int sz) elf_array->size += sz; } +#if ELF_IS_64 == 0 +static void elf_write_section_header(const elf32_shdr_t *shdr) +{ + elf_write_blk(elf_section_header, shdr, sizeof(*shdr)); +} +#endif + /* The dynamic-linking tables differ only in width between the two ELF classes, * so the generator below writes them through these helpers rather than * memcpy-ing a struct: shecc has no 64-bit integer type, so an ELF64 entry @@ -221,49 +236,56 @@ strbuf_t *elf_relplt_buf(void) return dynamic_sections.elf_relplt; } +static void elf_write_file_header(int is_64, + int entry, + int phnum, + int shoff, + int shnum, + int shstrndx) +{ + int header_size = is_64 ? 64 : sizeof(elf32_hdr_t); + int program_header_size = is_64 ? 56 : sizeof(elf32_phdr_t); + + elf_write_byte(elf_header, 0x7f); + elf_write_str(elf_header, "ELF"); + elf_write_byte(elf_header, is_64 ? 2 : 1); + elf_write_byte(elf_header, 1); + elf_write_byte(elf_header, 1); + for (int i = 0; i < 9; i++) + elf_write_byte(elf_header, 0); + + elf_write_short(elf_header, 2); + elf_write_short(elf_header, ELF_MACHINE); + elf_write_int(elf_header, 1); + if (is_64) { + elf_write_quad(elf_header, entry); + elf_write_quad(elf_header, header_size); + elf_write_quad(elf_header, 0); + } else { + elf_write_int(elf_header, entry); + elf_write_int(elf_header, header_size); + elf_write_int(elf_header, shoff); + } + elf_write_int(elf_header, ELF_FLAGS); + elf_write_short(elf_header, header_size); + elf_write_short(elf_header, program_header_size); + elf_write_short(elf_header, phnum); + elf_write_short(elf_header, is_64 ? 0 : sizeof(elf32_shdr_t)); + elf_write_short(elf_header, shnum); + elf_write_short(elf_header, shstrndx); +} + void elf_generate_header(void) { /* Check for null pointers to prevent crashes */ if (!elf_code || !elf_data || !elf_symtab || !elf_strtab || !elf_header) { fatal("ELF buffers not initialized"); - return; } -#if ELF_IS_64 == 1 - /* ELF64 executable header, 64 bytes, little-endian. - * - * No section headers are emitted, so e_shoff, e_shnum and e_shstrndx are - * all zero: a loader needs only the program headers to execute the image, - * and the dynamic loader reads PT_DYNAMIC rather than the section table. - */ - int phnum64 = 2; - if (dynlink) - phnum64 = 4; /* the two PT_LOADs plus PT_INTERP and PT_DYNAMIC */ - elf_write_byte(elf_header, 0x7f); - elf_write_str(elf_header, "ELF"); - elf_write_byte(elf_header, 2); /* EI_CLASS = ELFCLASS64 */ - elf_write_byte(elf_header, 1); /* EI_DATA = ELFDATA2LSB */ - elf_write_byte(elf_header, 1); /* EI_VERSION = EV_CURRENT */ - for (int i = 0; i < 9; i++) /* EI_OSABI, EI_ABIVERSION, padding */ - elf_write_byte(elf_header, 0); - - elf_write_short(elf_header, 2); /* e_type = ET_EXEC */ - elf_write_short(elf_header, ELF_MACHINE); /* e_machine */ - elf_write_int(elf_header, 1); /* e_version = EV_CURRENT */ - elf_write_quad(elf_header, elf_code_start + elf_entry_offset); /* e_entry */ - elf_write_quad(elf_header, 64); /* e_phoff: right after header */ - elf_write_quad(elf_header, 0); /* e_shoff: no section headers */ - elf_write_int(elf_header, ELF_FLAGS); /* e_flags */ - elf_write_short(elf_header, 64); /* e_ehsize */ - elf_write_short(elf_header, 56); /* e_phentsize */ - elf_write_short(elf_header, phnum64); /* e_phnum */ - elf_write_short(elf_header, 0); /* e_shentsize */ - elf_write_short(elf_header, 0); /* e_shnum */ - elf_write_short(elf_header, 0); /* e_shstrndx */ -#else - elf32_hdr_t hdr; - int phnum, shnum, shstrndx, shoff; + int phnum = dynlink ? 4 : 2; + int shnum = 0, shstrndx = 0, shoff = 0; +#if ELF_IS_64 == 0 if (dynlink) { /* In dynamic linking mode: * - number of program headers = 4 @@ -273,7 +295,6 @@ void elf_generate_header(void) int elf_relplt_size = dynamic_sections.use_relaplt ? dynamic_sections.elf_relaplt->size : dynamic_sections.elf_relplt->size; - phnum = 4; shnum = 15; shstrndx = 14; shoff = elf_header_len + elf_code->size + elf_data->size + @@ -290,7 +311,6 @@ void elf_generate_header(void) * - number of section headers = 8 * - section header index of .shstrtab = 7 */ - phnum = 2; shnum = 8; shstrndx = 7; @@ -348,238 +368,86 @@ void elf_generate_header(void) * ---+----------------+-------------------------------------------------+ * 34 | | | */ - /* ELF file header */ - hdr.e_ident[0] = (char) 0x7F; /* ELF magic number */ - hdr.e_ident[1] = 'E'; - hdr.e_ident[2] = 'L'; - hdr.e_ident[3] = 'F'; - hdr.e_ident[4] = 1; /* 32-bit */ - hdr.e_ident[5] = 1; /* little-endian */ - hdr.e_ident[6] = 1; /* ELF header version */ - hdr.e_ident[7] = 0; /* Target OS ABI */ - hdr.e_ident[8] = 0; /* ABI version */ - hdr.e_ident[9] = 0; /* Padding */ - hdr.e_ident[10] = 0; - hdr.e_ident[11] = 0; - hdr.e_ident[12] = 0; - hdr.e_ident[13] = 0; - hdr.e_ident[14] = 0; - hdr.e_ident[15] = 0; - hdr.e_type = 2; /* Object file type */ - hdr.e_machine = ELF_MACHINE; /* Instruction Set Architecture */ - hdr.e_version = 1; /* ELF version */ - hdr.e_entry = elf_code_start; /* entry point */ - hdr.e_phoff = sizeof(elf32_hdr_t); /* program header offset */ - hdr.e_shoff = shoff; /* section header offset */ - hdr.e_flags = ELF_FLAGS; /* flags */ - hdr.e_ehsize = sizeof(elf32_hdr_t); /* header size */ - hdr.e_phentsize = sizeof(elf32_phdr_t); /* program header size */ - hdr.e_phnum = phnum; /* number of program headers */ - hdr.e_shentsize = sizeof(elf32_shdr_t); /* section header size */ - hdr.e_shnum = shnum; /* number of section headers */ - hdr.e_shstrndx = shstrndx; /* section index with names */ - elf_write_blk(elf_header, &hdr, sizeof(elf32_hdr_t)); +#endif + /* ELF64 carries no section table; ELF32 includes its generated table. */ + elf_write_file_header( + ELF_IS_64, + ELF_IS_64 ? elf_code_start + elf_entry_offset : elf_code_start, phnum, + shoff, shnum, shstrndx); +} + +static void elf_write_program_header(int type, + int flags, + int offset, + int address, + int file_size, + int memory_size, + int alignment) +{ +#if ELF_IS_64 == 1 + elf_write_int(elf_program_header, type); + elf_write_int(elf_program_header, flags); + elf_write_quad(elf_program_header, offset); + elf_write_quad(elf_program_header, address); + elf_write_quad(elf_program_header, address); + elf_write_quad(elf_program_header, file_size); + elf_write_quad(elf_program_header, memory_size); + elf_write_quad(elf_program_header, alignment); +#else + elf32_phdr_t phdr = {type, offset, address, address, + file_size, memory_size, flags, alignment}; + elf_write_blk(elf_program_header, &phdr, sizeof(phdr)); #endif } void elf_generate_program_headers(void) { const strbuf_t *elf_relplt = elf_relplt_buf(); + int ro_size; + int data_file_ofs; + int rw_vaddr; + int rw_filesz; + int rw_memsz; + int dyn_extra = 0; + if (!elf_program_header || !elf_code || !elf_data || !elf_rodata || - (dynlink && - (!dynamic_sections.elf_interp || !elf_relplt || - !dynamic_sections.elf_plt || !dynamic_sections.elf_got || - !dynamic_sections.elf_dynstr || !dynamic_sections.elf_dynsym || - !dynamic_sections.elf_dynamic))) { + !elf_dynamic_sections_ready(elf_relplt)) { fatal("ELF section buffers not initialized"); - return; } -#if ELF_IS_64 == 1 - /* Two ELF64 PT_LOAD segments, 56 bytes each. Field order differs from - * ELF32: p_flags sits immediately after p_type rather than before p_align. - */ - int ro_size = elf_header_len + elf_code->size + elf_rodata->size; - if (dynlink) - ro_size += elf_relplt->size + dynamic_sections.elf_plt->size; - - /* read-only, executable segment: headers + .text + .rodata */ - elf_write_int(elf_program_header, 1); /* p_type = PT_LOAD */ - elf_write_int(elf_program_header, 5); /* p_flags = R|X */ - elf_write_quad(elf_program_header, 0); /* p_offset */ - elf_write_quad(elf_program_header, ELF_START); /* p_vaddr */ - elf_write_quad(elf_program_header, ELF_START); /* p_paddr */ - elf_write_quad(elf_program_header, ro_size); /* p_filesz */ - elf_write_quad(elf_program_header, ro_size); /* p_memsz */ - elf_write_quad(elf_program_header, PAGESIZE); /* p_align */ - - /* read-write segment. Statically linked it holds .data (plus .bss, which - * occupies no file space) and starts at the next page boundary so that - * p_vaddr === p_offset (mod p_align), which the kernel enforces. - * Dynamically linked it begins at .interp and covers everything the loader - * needs, which elf_preprocess() has already placed a page clear of the - * read-only segment. - */ - int data_file_ofs = ALIGN_UP(ro_size, PAGESIZE); - int rw_vaddr = elf_data_start; - int rw_filesz = elf_data->size; - int rw_memsz = elf_data->size + elf_bss_size; - + ro_size = elf_header_len + elf_code->size + elf_rodata->size; if (dynlink) { - int dyn_extra = dynamic_sections.elf_interp->size + - dynamic_sections.elf_got->size + - dynamic_sections.elf_dynstr->size + - dynamic_sections.elf_dynsym->size + - dynamic_sections.elf_dynamic->size; - data_file_ofs = ro_size; - rw_vaddr = dynamic_sections.elf_interp_start; - rw_filesz += dyn_extra; - rw_memsz += dyn_extra; + ro_size += elf_relplt->size + dynamic_sections.elf_plt->size; + dyn_extra = dynamic_sections.elf_interp->size + + dynamic_sections.elf_got->size + + dynamic_sections.elf_dynstr->size + + dynamic_sections.elf_dynsym->size + + dynamic_sections.elf_dynamic->size; } + data_file_ofs = dynlink ? ro_size : ALIGN_UP(ro_size, PAGESIZE); + rw_vaddr = dynlink ? dynamic_sections.elf_interp_start : elf_data_start; + rw_filesz = elf_data->size + dyn_extra; + rw_memsz = elf_data->size + elf_bss_size + dyn_extra; - elf_write_int(elf_program_header, 1); /* p_type = PT_LOAD */ - elf_write_int(elf_program_header, 6); /* p_flags = R|W */ - elf_write_quad(elf_program_header, data_file_ofs); /* p_offset */ - elf_write_quad(elf_program_header, rw_vaddr); - elf_write_quad(elf_program_header, rw_vaddr); - elf_write_quad(elf_program_header, rw_filesz); - elf_write_quad(elf_program_header, rw_memsz); - elf_write_quad(elf_program_header, PAGESIZE); + elf_write_program_header(1, 5, 0, ELF_START, ro_size, ro_size, PAGESIZE); + elf_write_program_header(1, 6, data_file_ofs, rw_vaddr, rw_filesz, rw_memsz, + PAGESIZE); if (dynlink) { - /* PT_INTERP: names the dynamic loader. */ - elf_write_int(elf_program_header, 3); /* p_type = PT_INTERP */ - elf_write_int(elf_program_header, 4); /* p_flags = R */ - elf_write_quad(elf_program_header, ro_size); - elf_write_quad(elf_program_header, dynamic_sections.elf_interp_start); - elf_write_quad(elf_program_header, dynamic_sections.elf_interp_start); - elf_write_quad(elf_program_header, strlen(DYN_LINKER) + 1); - elf_write_quad(elf_program_header, strlen(DYN_LINKER) + 1); - elf_write_quad(elf_program_header, 1); - - /* PT_DYNAMIC: the table the loader walks. */ + int interp_size = strlen(DYN_LINKER) + 1; int dynamic_ofs = ro_size + dynamic_sections.elf_interp->size + dynamic_sections.elf_got->size + dynamic_sections.elf_dynstr->size + dynamic_sections.elf_dynsym->size; - int dynamic_addr = elf_dynamic_start(); - elf_write_int(elf_program_header, 2); /* p_type = PT_DYNAMIC */ - elf_write_int(elf_program_header, 6); /* p_flags = R|W */ - elf_write_quad(elf_program_header, dynamic_ofs); - elf_write_quad(elf_program_header, dynamic_addr); - elf_write_quad(elf_program_header, dynamic_addr); - elf_write_quad(elf_program_header, dynamic_sections.elf_dynamic->size); - elf_write_quad(elf_program_header, dynamic_sections.elf_dynamic->size); - elf_write_quad(elf_program_header, 8); - } -#else - elf32_phdr_t phdr; - - /* Explain the meaning of each field in the ELF32 program header. - * - * | Program | | - * & | Header bytes | Explanation | - * ---+----------------+-------------------------------------------------+ - * 34 | 01 00 00 00 | p_type: Segment type; 1 -> loadable. | - * | 54 00 00 00 | p_offset: Offset of segment in the file. | - * | 54 00 01 00 | p_vaddr: Virtual address of loaded segment. | - * | 54 00 01 00 | p_paddr: Only used on systems where physical | - * | | address is relevant. | - * | 48 8a 03 00 | p_filesz: Size of the segment in the file image.| - * | 48 8a 03 00 | p_memsz: Size of the segment in memory. | - * | | This value should be greater than or | - * | | equal to p_filesz. | - * | 07 00 00 00 | p_flags: Segment-wise permissions; | - * | | 0x1 -> execute, 0x2 -> write, | - * | | 0x4 -> read | - * | 04 00 00 00 | p_align: Align segment to the specified value. | - * ---+----------------+-------------------------------------------------+ - * 54 | | | - */ - /* program header - read-only segment */ - phdr.p_type = 1; /* PT_LOAD */ - phdr.p_offset = 0; /* offset of segment */ - phdr.p_vaddr = ELF_START; /* virtual address */ - phdr.p_paddr = ELF_START; /* physical address */ - phdr.p_filesz = - elf_header_len + elf_code->size + elf_rodata->size; /* size in file */ - phdr.p_memsz = - elf_header_len + elf_code->size + elf_rodata->size; /* size in memory */ - phdr.p_flags = 5; /* flags */ - phdr.p_align = PAGESIZE; /* alignment */ - if (dynlink) { - phdr.p_filesz += elf_relplt->size + dynamic_sections.elf_plt->size; - phdr.p_memsz += elf_relplt->size + dynamic_sections.elf_plt->size; - } - elf_write_blk(elf_program_header, &phdr, sizeof(elf32_phdr_t)); - - /* program header - readable and writable segment */ - phdr.p_type = 1; /* PT_LOAD */ - phdr.p_offset = elf_header_len + elf_code->size + - elf_rodata->size; /* offset of segment */ - if (!dynlink) - phdr.p_offset = ALIGN_UP(phdr.p_offset, PAGESIZE); - phdr.p_vaddr = elf_data_start; /* virtual address */ - phdr.p_paddr = elf_data_start; /* physical address */ - phdr.p_filesz = elf_data->size; /* size in file */ - phdr.p_memsz = elf_data->size + elf_bss_size; /* size in memory */ - phdr.p_flags = 6; /* flags */ - phdr.p_align = PAGESIZE; /* alignment */ - if (dynlink) { - phdr.p_offset += elf_relplt->size + dynamic_sections.elf_plt->size; - phdr.p_vaddr = dynamic_sections.elf_interp_start; - phdr.p_paddr = dynamic_sections.elf_interp_start; - phdr.p_filesz += dynamic_sections.elf_interp->size + - dynamic_sections.elf_got->size + - dynamic_sections.elf_dynstr->size + - dynamic_sections.elf_dynsym->size + - dynamic_sections.elf_dynamic->size; - phdr.p_memsz += dynamic_sections.elf_interp->size + - dynamic_sections.elf_got->size + - dynamic_sections.elf_dynstr->size + - dynamic_sections.elf_dynsym->size + - dynamic_sections.elf_dynamic->size; - } - elf_write_blk(elf_program_header, &phdr, sizeof(elf32_phdr_t)); - - if (dynlink) { - /* program header - program interpreter (.interp section) */ - phdr.p_type = 3; /* PT_INTERP */ - phdr.p_offset = elf_header_len + elf_code->size + elf_rodata->size + - elf_relplt->size + - dynamic_sections.elf_plt->size; /* offset of segment */ - phdr.p_vaddr = dynamic_sections.elf_interp_start; /* virtual address */ - phdr.p_paddr = dynamic_sections.elf_interp_start; /* physical address */ - phdr.p_filesz = strlen(DYN_LINKER) + 1; /* size in file */ - phdr.p_memsz = strlen(DYN_LINKER) + 1; /* size in memory */ - phdr.p_flags = 4; /* flags */ - phdr.p_align = 1; /* alignment */ - elf_write_blk(elf_program_header, &phdr, sizeof(elf32_phdr_t)); - - /* program header - .dynamic section */ - phdr.p_type = 2; /* PT_DYNAMIC */ - phdr.p_offset = - elf_header_len + elf_code->size + elf_rodata->size + - elf_relplt->size + dynamic_sections.elf_plt->size + - dynamic_sections.elf_interp->size + dynamic_sections.elf_got->size + - dynamic_sections.elf_dynstr->size + - dynamic_sections.elf_dynsym->size; /* offset of segment */ - phdr.p_vaddr = dynamic_sections.elf_got_start + - dynamic_sections.elf_got->size + - dynamic_sections.elf_dynstr->size + - dynamic_sections.elf_dynsym->size; /* virtual address */ - phdr.p_paddr = dynamic_sections.elf_got_start + - dynamic_sections.elf_got->size + - dynamic_sections.elf_dynstr->size + - dynamic_sections.elf_dynsym->size; /* physical address */ - phdr.p_filesz = dynamic_sections.elf_dynamic->size; /* size in file */ - phdr.p_memsz = dynamic_sections.elf_dynamic->size; /* size in memory */ - phdr.p_flags = 6; /* flags */ - phdr.p_align = 4; /* alignment */ - elf_write_blk(elf_program_header, &phdr, sizeof(elf32_phdr_t)); + elf_write_program_header(3, 4, ro_size, + dynamic_sections.elf_interp_start, interp_size, + interp_size, 1); + elf_write_program_header(2, 6, dynamic_ofs, elf_dynamic_start(), + dynamic_sections.elf_dynamic->size, + dynamic_sections.elf_dynamic->size, + ELF_IS_64 ? 8 : 4); } -#endif } void elf_generate_section_headers(void) @@ -596,19 +464,23 @@ void elf_generate_section_headers(void) /* Check for null pointers to prevent crashes */ if (!elf_section_header || !elf_code || !elf_data || !elf_rodata || !elf_symtab || !elf_strtab || !elf_shstrtab || - (dynlink && - (!dynamic_sections.elf_interp || !elf_relplt || - !dynamic_sections.elf_plt || !dynamic_sections.elf_got || - !dynamic_sections.elf_dynstr || !dynamic_sections.elf_dynsym || - !dynamic_sections.elf_dynamic))) { + !elf_dynamic_sections_ready(elf_relplt)) { fatal("ELF section buffers not initialized"); - return; } /* section header table */ elf32_shdr_t shdr; int ofs = elf_header_len, sh_name = 0; +#define WRITE_FILE_SECTION(name, type, flags, addr, size, ...) \ + do { \ + elf32_shdr_t file_section = {sh_name, type, flags, addr, \ + ofs, size, __VA_ARGS__}; \ + elf_write_section_header(&file_section); \ + ofs += file_section.sh_size; \ + sh_name += sizeof(name); \ + } while (0) + /* The following table uses the text section header as an example to explain * the ELF32 section header. * @@ -636,255 +508,103 @@ void elf_generate_section_headers(void) * | | | */ /* NULL section */ - shdr.sh_name = sh_name; - shdr.sh_type = 0; - shdr.sh_flags = 0; - shdr.sh_addr = 0; - shdr.sh_offset = 0; - shdr.sh_size = 0; - shdr.sh_link = 0; - shdr.sh_info = 0; - shdr.sh_addralign = 0; - shdr.sh_entsize = 0; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); + shdr = (elf32_shdr_t) {sh_name, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + elf_write_section_header(&shdr); sh_name += 1; /* .text */ - shdr.sh_name = sh_name; - shdr.sh_type = 1; - shdr.sh_flags = 7; - shdr.sh_addr = elf_code_start; - shdr.sh_offset = ofs; - shdr.sh_size = elf_code->size; - shdr.sh_link = 0; - shdr.sh_info = 0; - shdr.sh_addralign = 4; - shdr.sh_entsize = 0; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); - ofs += elf_code->size; - sh_name += strlen(".text") + 1; + WRITE_FILE_SECTION(".text", 1, 7, elf_code_start, elf_code->size, 0, 0, 4, + 0); /* .rodata */ - shdr.sh_name = sh_name; /* Offset in shstrtab for ".rodata" */ - shdr.sh_type = 1; /* SHT_PROGBITS */ - shdr.sh_flags = 2; /* SHF_ALLOC only (read-only) */ - shdr.sh_addr = elf_rodata_start; - shdr.sh_offset = ofs; - shdr.sh_size = elf_rodata->size; - shdr.sh_link = 0; - shdr.sh_info = 0; - shdr.sh_addralign = 4; - shdr.sh_entsize = 0; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); - ofs += elf_rodata->size; - sh_name += strlen(".rodata") + 1; + WRITE_FILE_SECTION(".rodata", 1, 2, elf_rodata_start, elf_rodata->size, 0, + 0, 4, 0); if (dynlink) { /* .rel.plt or .rela.plt */ - int sh_type, sh_addr, sh_size, sh_entsize, __ofs, __sh_name; - - if (dynamic_sections.use_relaplt) { - sh_type = 4; /* SHT_RELA */ - sh_addr = dynamic_sections.elf_relaplt_start; - sh_size = dynamic_sections.elf_relaplt->size; - sh_entsize = sizeof(elf32_rela_t); - __ofs = dynamic_sections.elf_relaplt->size; - __sh_name = strlen(".rela.plt") + 1; - } else { - sh_type = 9; /* SHT_REL */ - sh_addr = dynamic_sections.elf_relplt_start; - sh_size = dynamic_sections.elf_relplt->size; - sh_entsize = sizeof(elf32_rel_t); - __ofs = dynamic_sections.elf_relplt->size; - __sh_name = strlen(".rel.plt") + 1; - } - shdr.sh_name = sh_name; - shdr.sh_type = sh_type; - shdr.sh_flags = 0x42; /* 0x40 | SHF_ALLOC */ - shdr.sh_addr = sh_addr; - shdr.sh_offset = ofs; - shdr.sh_size = sh_size; - shdr.sh_link = 8; /* The section header index of .dynsym. */ - shdr.sh_info = 6; /* The section header index of .got. */ - shdr.sh_addralign = 4; - shdr.sh_entsize = sh_entsize; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); - ofs += __ofs; - sh_name += __sh_name; + bool rela = dynamic_sections.use_relaplt; + shdr = (elf32_shdr_t) {sh_name, + rela ? 4 : 9, + 0x42, + rela ? dynamic_sections.elf_relaplt_start + : dynamic_sections.elf_relplt_start, + ofs, + elf_relplt->size, + 8, + 6, + 4, + elf_jmprel_size()}; + elf_write_section_header(&shdr); + ofs += elf_relplt->size; + sh_name += rela ? sizeof(".rela.plt") : sizeof(".rel.plt"); /* .plt */ - shdr.sh_name = sh_name; - shdr.sh_type = 1; - shdr.sh_flags = 0x6; - shdr.sh_addr = dynamic_sections.elf_plt_start; - shdr.sh_offset = ofs; - shdr.sh_size = dynamic_sections.elf_plt->size; - shdr.sh_link = 0; - shdr.sh_info = 0; - shdr.sh_addralign = 4; - shdr.sh_entsize = 4; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); - ofs += dynamic_sections.elf_plt->size; - sh_name += strlen(".plt") + 1; + WRITE_FILE_SECTION(".plt", 1, 0x6, dynamic_sections.elf_plt_start, + dynamic_sections.elf_plt->size, 0, 0, 4, 4); /* .interp */ - shdr.sh_name = sh_name; - shdr.sh_type = 1; - shdr.sh_flags = 0x2; - shdr.sh_addr = dynamic_sections.elf_interp_start; - shdr.sh_offset = ofs; - shdr.sh_size = strlen(DYN_LINKER) + 1; - shdr.sh_link = 0; - shdr.sh_info = 0; - shdr.sh_addralign = 1; - shdr.sh_entsize = 0; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); + shdr = (elf32_shdr_t) {sh_name, 1, + 0x2, dynamic_sections.elf_interp_start, + ofs, strlen(DYN_LINKER) + 1, + 0, 0, + 1, 0}; + elf_write_section_header(&shdr); ofs += dynamic_sections.elf_interp->size; sh_name += strlen(".interp") + 1; /* .got */ - shdr.sh_name = sh_name; - shdr.sh_type = 1; - shdr.sh_flags = 0x3; - shdr.sh_addr = dynamic_sections.elf_got_start; - shdr.sh_offset = ofs; - shdr.sh_size = dynamic_sections.elf_got->size; - shdr.sh_link = 0; - shdr.sh_info = 0; - shdr.sh_addralign = 4; - shdr.sh_entsize = PTR_SIZE; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); - ofs += dynamic_sections.elf_got->size; - sh_name += strlen(".got") + 1; + WRITE_FILE_SECTION(".got", 1, 0x3, dynamic_sections.elf_got_start, + dynamic_sections.elf_got->size, 0, 0, 4, PTR_SIZE); /* .dynstr */ - shdr.sh_name = sh_name; - shdr.sh_type = 3; - shdr.sh_flags = 0x2; - shdr.sh_addr = - dynamic_sections.elf_got_start + dynamic_sections.elf_got->size; - shdr.sh_offset = ofs; - shdr.sh_size = dynamic_sections.elf_dynstr->size; - shdr.sh_link = 0; - shdr.sh_info = 0; - shdr.sh_addralign = 1; - shdr.sh_entsize = 0; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); - ofs += dynamic_sections.elf_dynstr->size; - sh_name += strlen(".dynstr") + 1; + WRITE_FILE_SECTION( + ".dynstr", 3, 0x2, + dynamic_sections.elf_got_start + dynamic_sections.elf_got->size, + dynamic_sections.elf_dynstr->size, 0, 0, 1, 0); /* .dynsym */ - shdr.sh_name = sh_name; - shdr.sh_type = 11; - shdr.sh_flags = 0x2; - shdr.sh_addr = dynamic_sections.elf_got_start + - dynamic_sections.elf_got->size + - dynamic_sections.elf_dynstr->size; - shdr.sh_offset = ofs; - shdr.sh_size = dynamic_sections.elf_dynsym->size; - shdr.sh_link = 7; /* The section header index of .dynstr. */ - shdr.sh_info = 1; /* The index of the first non-local symbol. */ - shdr.sh_addralign = 4; - shdr.sh_entsize = sizeof(elf32_sym_t); - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); - ofs += dynamic_sections.elf_dynsym->size; - sh_name += strlen(".dynsym") + 1; + WRITE_FILE_SECTION( + ".dynsym", 11, 0x2, + dynamic_sections.elf_got_start + dynamic_sections.elf_got->size + + dynamic_sections.elf_dynstr->size, + dynamic_sections.elf_dynsym->size, 7, 1, 4, sizeof(elf32_sym_t)); /* .dynamic */ - shdr.sh_name = sh_name; - shdr.sh_type = 6; - shdr.sh_flags = 0x3; - shdr.sh_addr = dynamic_sections.elf_got_start + - dynamic_sections.elf_got->size + - dynamic_sections.elf_dynstr->size + - dynamic_sections.elf_dynsym->size; - shdr.sh_offset = ofs; - shdr.sh_size = dynamic_sections.elf_dynamic->size; - shdr.sh_link = 7; /* The section header index of .dynstr. */ - shdr.sh_info = 0; - shdr.sh_addralign = 4; - shdr.sh_entsize = 0; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); - ofs += dynamic_sections.elf_dynamic->size; - sh_name += strlen(".dynamic") + 1; + WRITE_FILE_SECTION(".dynamic", 6, 0x3, + dynamic_sections.elf_got_start + + dynamic_sections.elf_got->size + + dynamic_sections.elf_dynstr->size + + dynamic_sections.elf_dynsym->size, + dynamic_sections.elf_dynamic->size, 7, 0, 4, 0); } if (!dynlink) ofs = ALIGN_UP(ofs, PAGESIZE); /* .data */ - shdr.sh_name = sh_name; - shdr.sh_type = 1; - shdr.sh_flags = 3; - shdr.sh_addr = elf_data_start; - shdr.sh_offset = ofs; - shdr.sh_size = elf_data->size; - shdr.sh_link = 0; - shdr.sh_info = 0; - shdr.sh_addralign = 4; - shdr.sh_entsize = 0; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); - ofs += elf_data->size; - sh_name += strlen(".data") + 1; + WRITE_FILE_SECTION(".data", 1, 3, elf_data_start, elf_data->size, 0, 0, 4, + 0); /* .bss */ - shdr.sh_name = sh_name; /* Offset in shstrtab for ".bss" */ - shdr.sh_type = 8; /* SHT_NOBITS */ - shdr.sh_flags = 3; /* SHF_ALLOC | SHF_WRITE */ - shdr.sh_addr = elf_bss_start; - shdr.sh_offset = ofs; /* File offset (not actually used for NOBITS) */ - shdr.sh_size = elf_bss_size; - shdr.sh_link = 0; - shdr.sh_info = 0; - shdr.sh_addralign = 4; - shdr.sh_entsize = 0; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); + shdr = (elf32_shdr_t) {sh_name, 8, 3, elf_bss_start, ofs, elf_bss_size, 0, + 0, 4, 0}; + elf_write_section_header(&shdr); sh_name += strlen(".bss") + 1; /* Note: .bss is not written to file (SHT_NOBITS) */ /* .symtab */ - shdr.sh_name = sh_name; - shdr.sh_type = 2; - shdr.sh_flags = 0; - shdr.sh_addr = 0; - shdr.sh_offset = ofs; - shdr.sh_size = elf_symtab->size; - shdr.sh_link = dynlink ? 13 : 6; /* Link to .strtab */ - shdr.sh_info = elf_symbol_index; - shdr.sh_addralign = 4; - shdr.sh_entsize = 16; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); - ofs += elf_symtab->size; - sh_name += strlen(".symtab") + 1; + WRITE_FILE_SECTION(".symtab", 2, 0, 0, elf_symtab->size, dynlink ? 13 : 6, + elf_symbol_index, 4, 16); /* .strtab */ - shdr.sh_name = sh_name; - shdr.sh_type = 3; - shdr.sh_flags = 0; - shdr.sh_addr = 0; - shdr.sh_offset = ofs; - shdr.sh_size = elf_strtab->size; - shdr.sh_link = 0; - shdr.sh_info = 0; - shdr.sh_addralign = 1; - shdr.sh_entsize = 0; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); - ofs += elf_strtab->size; - sh_name += strlen(".strtab") + 1; + WRITE_FILE_SECTION(".strtab", 3, 0, 0, elf_strtab->size, 0, 0, 1, 0); /* .shstr */ - shdr.sh_name = sh_name; - shdr.sh_type = 3; - shdr.sh_flags = 0; - shdr.sh_addr = 0; - shdr.sh_offset = ofs; - shdr.sh_size = elf_shstrtab->size; - shdr.sh_link = 0; - shdr.sh_info = 0; - shdr.sh_addralign = 1; - shdr.sh_entsize = 0; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); + shdr = + (elf32_shdr_t) {sh_name, 3, 0, 0, ofs, elf_shstrtab->size, 0, 0, 1, 0}; + elf_write_section_header(&shdr); sh_name += strlen(".shstrtab") + 1; +#undef WRITE_FILE_SECTION #endif } @@ -896,7 +616,6 @@ void elf_align_to(strbuf_t *elf_array, int boundary) /* Check for null pointers to prevent crashes */ if (!elf_array) { fatal("ELF buffers not initialized for alignment"); - return; } while (elf_array->size & (boundary - 1)) @@ -1138,14 +857,8 @@ void elf_reset_dynamic_sections(void) void elf_generate_sections(void) { const strbuf_t *elf_relplt = elf_relplt_buf(); - if (!elf_shstrtab || - (dynlink && - (!dynamic_sections.elf_interp || !elf_relplt || - !dynamic_sections.elf_plt || !dynamic_sections.elf_got || - !dynamic_sections.elf_dynstr || !dynamic_sections.elf_dynsym || - !dynamic_sections.elf_dynamic))) { + if (!elf_shstrtab || !elf_dynamic_sections_ready(elf_relplt)) { fatal("ELF section buffers not initialized"); - return; } if (dynlink) @@ -1193,7 +906,6 @@ void elf_add_symbol(const char *symbol, int pc) /* Check for null pointers to prevent crashes */ if (!symbol || !elf_symtab || !elf_strtab) { fatal("Invalid parameters for elf_add_symbol"); - return; } elf_write_int(elf_symtab, elf_strtab->size); @@ -1206,20 +918,25 @@ void elf_add_symbol(const char *symbol, int pc) elf_symbol_index++; } -void elf_preprocess(void) +/* Bytes of file and program headers in front of the code, which starts right + * after them. + */ +int elf_header_size(void) { #if ELF_IS_64 == 1 /* ELF64 header (64) plus two program headers (56 each), and two more -- * PT_INTERP and PT_DYNAMIC -- when linking dynamically. */ - elf_header_len = 64 + (56 << 1); - if (dynlink) - elf_header_len += (56 << 1); + return 64 + (56 << 1) + (dynlink ? 56 << 1 : 0); #else - elf_header_len = sizeof(elf32_hdr_t) + (sizeof(elf32_phdr_t) << 1); - if (dynlink) - elf_header_len += (sizeof(elf32_phdr_t) << 1); + return sizeof(elf32_hdr_t) + (sizeof(elf32_phdr_t) << 1) + + (dynlink ? sizeof(elf32_phdr_t) << 1 : 0); #endif +} + +void elf_preprocess(void) +{ + elf_header_len = elf_header_size(); elf_align(elf_data); elf_align(elf_rodata); elf_code_start = ELF_START + elf_header_len; diff --git a/src/globals.c b/src/globals.c index 3ba7e0c9..7f0fad88 100644 --- a/src/globals.c +++ b/src/globals.c @@ -7,6 +7,7 @@ #pragma once #include +#include #include #include #include @@ -21,6 +22,8 @@ __noreturn void limit_error(char *msg); /* Forward declaration for string interning */ char *intern_string(char *str); +func_t *find_func(char *func_name); +__noreturn void fatal(const char *msg); /* Lexer */ token_t *cur_token; @@ -32,12 +35,54 @@ strbuf_t *LIBC_SRC; hashmap_t *SRC_FILE_MAP; hashmap_t *FUNC_MAP; -hashmap_t *CONSTANTS_MAP; /* Types */ type_t *TYPES; int types_idx = 0; +int builtin_types_idx; +static unsigned int internal_linkage_name_idx; + +static const char *file_scope_var_name(const var_t *var) +{ + return var->source_name ? var->source_name : var->var_name; +} + +static void assign_internal_linkage_name(var_t *var, const char *source_name) +{ + char name[48]; + + var->source_name = intern_string((char *) source_name); + snprintf(name, sizeof(name), "__shecc_internal_%u", + internal_linkage_name_idx++); + var->var_name = intern_string(name); +} + +static const char *function_source_name(const func_t *func) +{ + return file_scope_var_name(&func->return_def); +} + +typedef enum { + ORDINARY_NONE = 0, + ORDINARY_CONSTANT = 1, + ORDINARY_VARIABLE = 2, + ORDINARY_TYPEDEF = 4, + ORDINARY_PARAMETER = 8, + ORDINARY_ANY = 15 +} ordinary_kind_t; + +typedef struct tu_ordinary_binding { + ordinary_kind_t kind; + void *value; +} tu_ordinary_binding_t; + +static void register_tu_ordinary_name(const char *name, + ordinary_kind_t kind, + void *value); +static ordinary_kind_t lookup_tu_ordinary_name(const char *name, + int kinds, + void **value); type_t *TY_void; type_t *TY_char; @@ -58,15 +103,11 @@ type_t *TY_long_double; /* Arenas */ -arena_t *INSN_ARENA; - -/* HASHMAP_ARENA is responsible for hashmap_node_t allocation */ -arena_t *HASHMAP_ARENA; /* BLOCK_ARENA is responsible for block_t / var_t allocation */ arena_t *BLOCK_ARENA; -/* BB_ARENA is responsible for basic_block_t / ph2_ir_t allocation */ +/* BB_ARENA is responsible for basic_block_t / vir_mop_t allocation */ arena_t *BB_ARENA; /* TOKEN_ARENA is responsible for token_t (including literal) / @@ -74,23 +115,23 @@ arena_t *BB_ARENA; */ arena_t *TOKEN_ARENA; -/* GENERAL_ARENA is responsible for functions, symbols, constants, aliases, - * macros, and traversal args +/* GENERAL_ARENA is responsible for functions, symbols, constants, aliases, and + * macros */ arena_t *GENERAL_ARENA; -int bb_label_idx = 0; - -ph2_ir_t **PH2_IR_FLATTEN; -int ph2_ir_idx = 0; +vir_mop_t **MOP_FLAT; +int mop_flat_count = 0; +int mop_flat_capacity; func_list_t FUNC_LIST; func_t *GLOBAL_FUNC; block_t *GLOBAL_BLOCK; +block_t *CURRENT_TU_SCOPE; basic_block_t *MAIN_BB; int elf_offset = 0; +int translation_unit_count; -regfile_t REGS[REG_CNT]; hashmap_t *INCLUSION_MAP; @@ -114,19 +155,160 @@ int elf_bss_size; int elf_entry_offset = 0; dynamic_sections_t dynamic_sections; +/* C99 integer suffix order: [u][l|ll][u]. */ +bool numeric_suffix_is_valid(const char *suffix) +{ + int pos = 0; + + if ((suffix[pos] | 32) == 'u') + pos++; + if ((suffix[pos] | 32) == 'l') { + pos++; + if ((suffix[pos] | 32) == 'l') + pos++; + } + if ((suffix[pos] | 32) == 'u') + pos++; + return suffix[pos] == '\0'; +} + +/* The radix implied by an integer literal's leading characters. */ +int numeric_literal_base(const char *literal) +{ + if (literal[0] != '0') + return 10; + if ((literal[1] | 32) == 'x') + return 16; + if ((literal[1] | 32) == 'b') + return 2; + return 8; +} + +opcode_t operator_for_token(token_kind_t token) +{ + switch (token) { + case T_plus: + return OP_add; + case T_minus: + return OP_sub; + case T_bit_not: + return OP_bit_not; + case T_log_not: + return OP_log_not; + case T_asterisk: + return OP_mul; + case T_divide: + return OP_div; + case T_mod: + return OP_mod; + case T_lshift: + return OP_lshift; + case T_rshift: + return OP_rshift; + case T_log_and: + return OP_log_and; + case T_log_or: + return OP_log_or; + case T_eq: + return OP_eq; + case T_noteq: + return OP_neq; + case T_lt: + return OP_lt; + case T_le: + return OP_leq; + case T_gt: + return OP_gt; + case T_ge: + return OP_geq; + case T_ampersand: + return OP_bit_and; + case T_bit_or: + return OP_bit_or; + case T_bit_xor: + return OP_bit_xor; + case T_question: + return OP_ternary; + default: + return OP_generic; + } +} + +/* Shared by the parser and preprocessor's expression evaluator. */ +int get_operator_prio(opcode_t op) +{ + switch (op) { + case OP_ternary: + return 3; + case OP_log_or: + return 4; + case OP_log_and: + return 5; + case OP_bit_or: + return 6; + case OP_bit_xor: + return 7; + case OP_bit_and: + return 8; + case OP_eq: + case OP_neq: + return 9; + case OP_lt: + case OP_leq: + case OP_gt: + case OP_geq: + return 10; + case OP_lshift: + case OP_rshift: + return 11; + case OP_add: + case OP_sub: + return 12; + case OP_mul: + case OP_div: + case OP_mod: + return 13; + default: + return 0; + } +} + /* Command line compilation flags */ bool dynlink = false; bool imm_binding = false; bool libc = true; +bool optimize = true; bool expand_only = false; bool dump_ir = false; +bool dump_vir = false; bool dump_dot = false; bool hard_mul_div = false; bool warn_string_literals = false; bool strict_c99 = false; +bool dump_stats = false; char *include_dirs[MAX_INCLUDE_DIRS]; int include_dirs_idx = 0; +/* Allocation counters are a stable pre-VIR baseline even when later passes + * delete or rewrite the objects they counted. + */ +int stats_func_count = 0; +int stats_var_count = 0; +int stats_field_slot_count = 0; +int stats_bb_count = 0; +int stats_mop_count = 0; +static int stats_tu_index_lookups = 0; +static int stats_tu_index_registrations = 0; +static hashmap_t *TU_ORDINARY_MAP; + +/* Payload capacity currently retained by all live arenas, and its true + * allocation-time high water. This deliberately excludes arena headers, malloc + * metadata, and non-arena allocations. + */ +int stats_live_arena_capacity = 0; +int stats_peak_arena_capacity = 0; +int stats_retired_token_arena_peak = 0; + /* Create a new arena block with given capacity. * @capacity: The capacity of the arena block. Must be positive. * @@ -181,6 +363,10 @@ arena_t *arena_init(int initial_capacity) } arena->head = arena_block_create(initial_capacity); arena->total_bytes = initial_capacity; + arena->peak_bytes = initial_capacity; + stats_live_arena_capacity += initial_capacity; + if (stats_live_arena_capacity > stats_peak_arena_capacity) + stats_peak_arena_capacity = stats_live_arena_capacity; /* Use the initial capacity as the default block size for future growth. */ arena->block_size = initial_capacity; return arena; @@ -207,17 +393,22 @@ void *arena_alloc(arena_t *arena, int size) size = (size + alignment - 1) & ~(alignment - 1); if (!arena->head || arena->head->offset + size > arena->head->capacity) { - /* Need a new block: choose capacity = max(DEFAULT_ARENA_SIZE, - * arena->block_size, size) + /* Grow by the arena's own block size, or by @size when that is larger. + * A small arena growing in small blocks fits them into the holes the + * freed VIR graphs leave during lowering, where 256 KiB blocks took + * fresh pages and raised the peak by 10 MB. */ - const int base = - (arena->block_size > DEFAULT_ARENA_SIZE ? arena->block_size - : DEFAULT_ARENA_SIZE); + const int base = arena->block_size; const int new_capacity = (size > base ? size : base); arena_block_t *new_block = arena_block_create(new_capacity); new_block->next = arena->head; arena->head = new_block; arena->total_bytes += new_capacity; + stats_live_arena_capacity += new_capacity; + if (stats_live_arena_capacity > stats_peak_arena_capacity) + stats_peak_arena_capacity = stats_live_arena_capacity; + if (arena->total_bytes > arena->peak_bytes) + arena->peak_bytes = arena->total_bytes; } void *ptr = arena->head->memory + arena->head->offset; @@ -225,6 +416,59 @@ void *arena_alloc(arena_t *arena, int size) return ptr; } +/* Mark consumed arena payload without charging future allocations. The mark is + * valid while its block remains in the arena's newest-to-oldest chain. + */ +arena_mark_t arena_mark(arena_t *arena) +{ + arena_mark_t mark; + + mark.head = arena ? arena->head : NULL; + mark.offset = mark.head ? mark.head->offset : 0; + return mark; +} + +/* Return payload allocated since a mark, including new blocks and growth of its + * original head block. + * + * Return -1 if the marked block is no longer present. + */ +int arena_bytes_since(arena_t *arena, arena_mark_t mark) +{ + int bytes = 0; + arena_block_t *block = arena ? arena->head : NULL; + + while (block && block != mark.head) { + bytes += block->offset; + block = block->next; + } + if (block != mark.head) + return -1; + return bytes + (mark.head ? mark.head->offset - mark.offset : 0); +} + +bool arena_rewind(arena_t *arena, arena_mark_t mark) +{ + if (!arena || !mark.head) + return false; + + arena_block_t *block = arena->head; + while (block && block != mark.head) + block = block->next; + if (!block || mark.offset > mark.head->offset) + return false; + + while (arena->head != mark.head) { + block = arena->head; + arena->head = block->next; + arena->total_bytes -= block->capacity; + stats_live_arena_capacity -= block->capacity; + arena_block_free(block); + } + arena->head->offset = mark.offset; + return true; +} + /* arena_alloc() plus explicit zero‑initialization. * @arena: The arena to allocate memory from. Must not be NULL. * @n: Number of elements. @@ -330,12 +574,16 @@ char *arena_strdup(arena_t *arena, const char *str) /* Typed allocators for consistent memory management */ func_t *arena_alloc_func(void) { - return arena_calloc(GENERAL_ARENA, 1, sizeof(func_t)); + func_t *func = arena_calloc(GENERAL_ARENA, 1, sizeof(func_t)); + + stats_func_count++; + return func; } -symbol_t *arena_alloc_symbol(void) +basic_block_t *arena_alloc_bb(void) { - return arena_calloc(GENERAL_ARENA, 1, sizeof(symbol_t)); + stats_bb_count++; + return arena_calloc(BB_ARENA, 1, sizeof(basic_block_t)); } constant_t *arena_alloc_constant(void) @@ -347,14 +595,11 @@ constant_t *arena_alloc_constant(void) return c; } -bb_traversal_args_t *arena_alloc_traversal_args(void) -{ - /* Keep using calloc for safety */ - return arena_calloc(GENERAL_ARENA, 1, sizeof(bb_traversal_args_t)); -} - void arena_free(arena_t *arena) { + if (!arena) + return; + arena_block_t *block = arena->head; arena_block_t *next; @@ -364,18 +609,15 @@ void arena_free(arena_t *arena) block = next; } + stats_live_arena_capacity -= arena->total_bytes; free(arena); } -/* Hash a string with FNV-1a hash function and converts into usable hashmap - * index. The range of returned hashmap index is ranged from "(0 ~ - * 2,147,483,647) mod size" due to lack of unsigned integer implementation. - * @size: The size of map. Must not be negative or 0. +/* Hash a string with FNV-1a, folded to a non-negative int because there is no + * unsigned arithmetic to do it in; a map masks it down to its capacity. * @key: The key string. May be NULL. - * - * Return: The usable hashmap index. */ -int hashmap_hash_index(int size, char *key) +int hashmap_hash(char *key) { if (!key) return 0; @@ -388,7 +630,12 @@ int hashmap_hash_index(int size, char *key) } const int mask = hash >> 31; - return ((hash ^ mask) - mask) & (size - 1); + return (hash ^ mask) - mask; +} + +static bool hashmap_node_matches(hashmap_node_t *node, int hash, char *key) +{ + return node->hash == hash && (node->key == key || !strcmp(node->key, key)); } int round_up_pow2(int v) @@ -420,6 +667,7 @@ hashmap_t *hashmap_create(int cap) map->size = 0; map->cap = round_up_pow2(cap); + map->has_owned_keys = false; map->table = calloc(map->cap, sizeof(hashmap_node_t)); if (!map->table) { @@ -432,7 +680,7 @@ hashmap_t *hashmap_create(int cap) } -void hashmap_rehash(hashmap_t *map) +static void hashmap_rehash(hashmap_t *map) { if (!map) return; @@ -457,7 +705,7 @@ void hashmap_rehash(hashmap_t *map) char *key = old_table[i].key; void *val = old_table[i].val; - int index = hashmap_hash_index(map->cap, key); + int index = old_table[i].hash & (map->cap - 1); int start = index; while (map->table[index].occupied) { @@ -471,37 +719,36 @@ void hashmap_rehash(hashmap_t *map) map->table[index].key = key; map->table[index].val = val; + map->table[index].hash = old_table[i].hash; map->table[index].occupied = true; + map->table[index].owns_key = old_table[i].owns_key; map->size++; } } free(old_table); } -/* Put a key-value pair into given hashmap. If key already contains a value, - * then replace it with new value, the old value will be freed. - * @map: The hashmap to be put into. Must not be NULL. - * @key: The key string. May be NULL. - * @val: The value pointer. May be NULL. This value's lifetime is held by - * hashmap. +/* Find an existing key or the empty slot for a new one, growing the table + * before its load factor exceeds 50%. */ -void hashmap_put(hashmap_t *map, char *key, void *val) +static hashmap_node_t *hashmap_prepare_slot(hashmap_t *map, char *key) { + int hash; + int index; + int start; + if (!map) - return; + return NULL; - /* Check if size of map exceeds load factor 50% (or 1/2 of capacity) */ if ((map->cap >> 1) <= map->size) hashmap_rehash(map); - int index = hashmap_hash_index(map->cap, key); - int start = index; - + hash = hashmap_hash(key); + index = hash & (map->cap - 1); + start = index; while (map->table[index].occupied) { - if (!strcmp(map->table[index].key, key)) { - map->table[index].val = val; - return; - } + if (hashmap_node_matches(&map->table[index], hash, key)) + return &map->table[index]; index = (index + 1) & (map->cap - 1); if (index == start) { @@ -511,15 +758,60 @@ void hashmap_put(hashmap_t *map, char *key, void *val) } } - map->table[index].key = arena_strdup(HASHMAP_ARENA, key); - map->table[index].val = val; - map->table[index].occupied = true; - map->size++; + map->table[index].hash = hash; + return &map->table[index]; +} + +/* Put an owned copy of @key into the map. Replacing an existing value keeps its + * original key and ownership. + * @map: The hashmap to be put into. May be NULL. + * @key: Non-NULL key string. + * @val: The value pointer. May be NULL. The map does not own the value. + */ +static void hashmap_put_impl(hashmap_t *map, + char *key, + void *val, + bool copy_key) +{ + hashmap_node_t *node = hashmap_prepare_slot(map, key); + + if (!node) + return; + + if (!node->occupied) { + if (copy_key) { + size_t key_size = strlen(key) + 1; + char *stored_key = malloc(key_size); + + if (!stored_key) { + printf("Failed to allocate hashmap key\n"); + fflush(stdout); /* see fatal() */ + abort(); + } + memcpy(stored_key, key, key_size); + key = stored_key; + } + node->key = key; + node->occupied = true; + node->owns_key = copy_key; + map->has_owned_keys |= copy_key; + map->size++; + } + + node->val = val; +} + +/* The caller must provide a non-NULL key and keep it alive for the map's + * lifetime. + */ +void hashmap_put_borrowed(hashmap_t *map, char *key, void *val) +{ + hashmap_put_impl(map, key, val, false); } /* Get key-value pair node from hashmap from given key. - * @map: The hashmap to be looked up. Must no be NULL. - * @key: The key string. May be NULL. + * @map: The hashmap to be looked up. May be NULL. + * @key: Non-NULL key string. * * Return: The look up result, if the key-value pair entry exists, then returns * address of itself, NULL otherwise. @@ -529,11 +821,12 @@ hashmap_node_t *hashmap_get_node(hashmap_t *map, char *key) if (!map) return NULL; - int index = hashmap_hash_index(map->cap, key); + int hash = hashmap_hash(key); + int index = hash & (map->cap - 1); int start = index; while (map->table[index].occupied) { - if (!strcmp(map->table[index].key, key)) + if (hashmap_node_matches(&map->table[index], hash, key)) return &map->table[index]; index = (index + 1) & (map->cap - 1); @@ -545,8 +838,8 @@ hashmap_node_t *hashmap_get_node(hashmap_t *map, char *key) } /* Get value from hashmap from given key. - * @map: The hashmap to be looked up. Must no be NULL. - * @key: The key string. May be NULL. + * @map: The hashmap to be looked up. May be NULL. + * @key: Non-NULL key string. * * Return: The look up result, if the key-value pair entry exists, then returns * its value's address, NULL otherwise. @@ -558,8 +851,8 @@ void *hashmap_get(hashmap_t *map, char *key) } /* Check if the key-value pair entry exists from given key. - * @map: The hashmap to be looked up. Must no be NULL. - * @key: The key string. May be NULL. + * @map: The hashmap to be looked up. May be NULL. + * @key: Non-NULL key string. * * Return: The look up result, if the key-value pair entry exists, then returns * true, false otherwise. @@ -569,117 +862,118 @@ bool hashmap_contains(hashmap_t *map, char *key) return hashmap_get_node(map, key); } -/* Free the hashmap, this also frees key-value pair entry's value. - * @map: The hashmap to be looked up. Must no be NULL. +/* Free the map table, any owned key copies, and the header. Mapped values have + * separate lifetimes and are not freed here. + * @map: The hashmap to be freed. May be NULL. */ void hashmap_free(hashmap_t *map) { if (!map) return; + if (map->has_owned_keys) + for (int i = 0; i < map->cap; i++) + if (map->table[i].occupied && map->table[i].owns_key) + free(map->table[i].key); free(map->table); free(map); } -/* Find the type by the given name. - * @type_name: The name to be searched. - * @flag: - * 0 - Search in all type names. - * 1 - Search in all names, excluding the tags of structure. - * 2 - Only search in tags. - * - * Return: The pointer to the type, or NULL if not found. - */ -type_t *find_type(const char *type_name, int flag) +void reset_tu_ordinary_index(void) { - char head = type_name[0]; + hashmap_free(TU_ORDINARY_MAP); + TU_ORDINARY_MAP = hashmap_create(32); + if (!TU_ORDINARY_MAP) + fatal("Failed to allocate translation-unit ordinary-name index"); +} - for (int i = 0; i < types_idx; i++) { - if (TYPES[i].type_name[0] != head) - continue; - if (TYPES[i].base_type == TYPE_struct || - TYPES[i].base_type == TYPE_union) { - if (flag == 1) - continue; - if (!strcmp(TYPES[i].type_name, type_name)) - return &TYPES[i]; - } else { - if (flag == 2) - continue; - if (!strcmp(TYPES[i].type_name, type_name)) { - /* If it is a forwardly declared alias of a structure, return - * the base structure type. A function type alias with a void - * return is no such alias despite its zero size. - */ - type_t *alias = &TYPES[i]; - type_t *base = alias->base_struct; - - if (alias->base_type != TYPE_typedef || alias->size || - alias->ptr_level || alias->is_direct_function_type) - return alias; - - /* The base would drop the qualifiers of `typedef const struct S - * cs_t;`, so a qualified alias keeps its own descriptor and - * takes the layout once the tag has been completed. - */ - if (!base || (!alias->is_const_qualified && - !alias->is_volatile_qualified)) - return base; - if (!base->size) - return alias; - alias->size = base->size; - alias->alignment = base->alignment; - alias->fields = base->fields; - alias->num_fields = base->num_fields; - alias->is_union = base->is_union; - alias->has_flexible_array_member = - base->has_flexible_array_member; - return alias; - } - } - } - return NULL; +static void register_tu_ordinary_name(const char *name, + ordinary_kind_t kind, + void *value) +{ + tu_ordinary_binding_t *binding; + char *stable_name; + + if (!TU_ORDINARY_MAP) + fatal("Translation-unit ordinary-name index is not initialized"); + stable_name = intern_string((char *) name); + binding = hashmap_get(TU_ORDINARY_MAP, stable_name); + if (binding) + return; + binding = arena_alloc(BLOCK_ARENA, sizeof(*binding)); + binding->kind = kind; + binding->value = value; + hashmap_put_borrowed(TU_ORDINARY_MAP, stable_name, binding); + if (dump_stats) + stats_tu_index_registrations++; } -ph2_ir_t *add_existed_ph2_ir(ph2_ir_t *ph2_ir) +static ordinary_kind_t lookup_tu_ordinary_name(const char *name, + int kinds, + void **value) { - if (ph2_ir_idx >= MAX_IR_INSTR) - limit_error("too many phase-2 IR instructions"); - PH2_IR_FLATTEN[ph2_ir_idx++] = ph2_ir; - return ph2_ir; + tu_ordinary_binding_t *binding; + + if (dump_stats) + stats_tu_index_lookups++; + binding = + TU_ORDINARY_MAP ? hashmap_get(TU_ORDINARY_MAP, (char *) name) : NULL; + if (!binding || !(kinds & binding->kind)) + return ORDINARY_NONE; + if (value) + *value = binding->value; + return binding->kind; } -ph2_ir_t *add_ph2_ir(opcode_t op) +static type_t *resolve_type_alias(type_t *alias) { - ph2_ir_t *ph2_ir = arena_alloc(BB_ARENA, sizeof(ph2_ir_t)); - ph2_ir->op = op; - /* Initialize all fields explicitly */ - ph2_ir->next = NULL; - ph2_ir->is_branch_detached = 0; - ph2_ir->src0 = 0; - ph2_ir->src1 = 0; + type_t *base = alias->base_struct; + + if (alias->base_type != TYPE_typedef || alias->size || alias->ptr_level || + alias->is_direct_function_type) + return alias; - /* Only a select names a third source, but the allocation is not zeroed and - * every field is set here by hand. + /* A zero-sized alias can be a forward declaration of a record, but a + * function type with a void return is also zero-sized and is not an alias + * to its return type. */ - ph2_ir->src2 = 0; - ph2_ir->dest = 0; - ph2_ir->func_name = NULL; - ph2_ir->next_bb = NULL; - ph2_ir->then_bb = NULL; - ph2_ir->else_bb = NULL; - ph2_ir->ofs_based_on_stack_top = false; - - /* Default to the full slot. Slots are PTR_SIZE wide, so a wide access is - * always valid; only an address-taken narrow scalar may be written behind - * the allocator's back, and reg-alloc narrows those explicitly. + if (!base || (!alias->is_const_qualified && !alias->is_volatile_qualified)) + return base; + if (!base->size) + return alias; + + /* Keep qualifiers on the alias while filling in the completed tag's layout. */ - ph2_ir->size_bytes = PTR_SIZE; - ph2_ir->is_pointer = false; - ph2_ir->src0_is_pointer = false; - ph2_ir->src1_is_pointer = false; - ph2_ir->is_volatile = false; - return add_existed_ph2_ir(ph2_ir); + alias->size = base->size; + alias->alignment = base->alignment; + alias->fields = base->fields; + alias->num_fields = base->num_fields; + alias->is_union = base->is_union; + alias->has_flexible_array_member = base->has_flexible_array_member; + return alias; +} + +/* Reject mismatched prefixes before strcmp. */ +static bool identifier_matches(const char *name, const char *query) +{ + return name[0] == query[0] && (!query[0] || name[1] == query[1]) && + !strcmp(name, query); +} + +/* Builtin types are initialized once before translation units are parsed. User + * typedefs share TYPES for their descriptors, but must be reached only through + * their translation-unit/lexical bindings. + */ +static type_t *find_builtin_type(const char *name) +{ + for (int i = 0; i < builtin_types_idx; i++) { + type_t *type = &TYPES[i]; + + if (type->base_type != TYPE_struct && type->base_type != TYPE_union && + identifier_matches(type->type_name, name)) + return type; + } + return NULL; } block_t *add_block(block_t *parent, func_t *func) @@ -729,7 +1023,7 @@ char *intern_string(char *str) interned = arena_alloc(GENERAL_ARENA, len); strcpy(interned, str); - hashmap_put(string_pool->strings, interned, interned); + hashmap_put_borrowed(string_pool->strings, interned, interned); return interned; } @@ -804,6 +1098,126 @@ bool hex_escape_exceeds_byte(const char *text) return false; } +/* Decode one escape beginning at text[*input]. Narrow strings mask hex runs to + * a byte; wide strings retain their value as one int unit. + */ +static bool decode_escape_value(const char *text, + int *input, + bool wide, + unsigned int *value, + bool *is_ucn) +{ + int i = *input; + + *is_ucn = false; + switch (text[i]) { + case 'a': + *value = '\a'; + i++; + break; + case 'b': + *value = '\b'; + i++; + break; + case 'f': + *value = '\f'; + i++; + break; + case 'n': + *value = '\n'; + i++; + break; + case 'r': + *value = '\r'; + i++; + break; + case 't': + *value = '\t'; + i++; + break; + case 'v': + *value = '\v'; + i++; + break; + case '\\': + *value = '\\'; + i++; + break; + case '\'': + *value = '\''; + i++; + break; + case '"': + *value = '"'; + i++; + break; + case '?': + *value = '?'; + i++; + break; + case 'e': + if (strict_c99) + return false; + *value = 27; + i++; + break; + case 'x': + i++; + if (!isxdigit((unsigned char) text[i])) + return false; + *value = 0; + while (isxdigit((unsigned char) text[i])) { + int digit = hex_digit_value(text[i]); + + if (wide) { + if (*value > 0x07ffffffU) + return false; + *value = (*value << 4) + digit; + } else + *value = ((*value << 4) + digit) & 0xff; + i++; + } + if (wide && *value > 0x7fffffffU) + return false; + break; + case 'u': + case 'U': { + int digits = text[i] == 'u' ? 4 : 8; + + *value = 0; + i++; + for (int n = 0; n < digits; n++) { + if (!isxdigit((unsigned char) text[i]) || + (wide && *value > 0x07ffffffU)) + return false; + *value = (*value << 4) + hex_digit_value(text[i++]); + } + + /* C99 excludes surrogates, out-of-range values, and basic-source + * characters other than $, @, and ` from universal escapes. + */ + if ((wide && *value > 0x7fffffffU) || *value > 0x10ffff || + (*value >= 0xd800 && *value <= 0xdfff) || + (*value < 0xa0 && *value != '$' && *value != '@' && *value != '`')) + return false; + *is_ucn = true; + break; + } + default: + if (!text[i]) + return false; + if (text[i] >= '0' && text[i] <= '7') { + *value = 0; + for (int n = 0; n < 3 && text[i] >= '0' && text[i] <= '7'; n++) + *value = *value * 8 + (text[i++] - '0'); + } else + *value = (unsigned char) text[i++]; + break; + } + *input = i; + return true; +} + int unescape_string(const char *input, char *output, int output_size) { if (!input || !output || output_size == 0) @@ -813,160 +1227,31 @@ int unescape_string(const char *input, char *output, int output_size) while (input[i] != '\0' && j < output_size - 1) { if (input[i] != '\\') { - /* Regular characters */ output[j++] = input[i++]; continue; } - i++; - switch (input[i]) { - case 'a': - output[j++] = '\a'; - i++; - break; - case 'b': - output[j++] = '\b'; - i++; - break; - case 'f': - output[j++] = '\f'; - i++; - break; - case 'e': - if (strict_c99) { - output[j] = '\0'; - return -1; - } - output[j++] = 27; - i++; - break; - case 'n': - output[j++] = '\n'; - i++; - break; - case 'r': - output[j++] = '\r'; - i++; - break; - case 't': - output[j++] = '\t'; - i++; - break; - case 'v': - output[j++] = '\v'; - i++; - break; - case '\\': - output[j++] = '\\'; - i++; - break; - case '\'': - output[j++] = '\''; - i++; - break; - case '"': - output[j++] = '"'; - i++; - break; - case '?': - output[j++] = '\?'; - i++; - break; - case 'x': { - /* C99 hexadecimal escapes consume the complete run of hex digits, - * unlike octal escapes which are limited to three. - */ - i++; /* Skips 'x' */ - - if (!isxdigit(input[i])) { - /* Terminate before bailing: callers read output[0], and an - * unterminated buffer left them reading stack garbage. - */ - output[j] = '\0'; - return -1; - } - - /* Only the low byte reaches the output, so keep only that much - * rather than shift a long digit run out of range. A narrow literal - * whose value does not fit has already been diagnosed by - * hex_escape_exceeds_byte(); a wide one takes its value from - * decode_wstring_units() instead. - */ - unsigned int value = 0; - while (isxdigit(input[i])) { - value = ((value << 4) + hex_digit_value(input[i])) & 0xff; - i++; - } + unsigned int value; + bool is_ucn; - output[j++] = (char) value; - break; + if (!decode_escape_value(input, &i, false, &value, &is_ucn)) { + output[j] = '\0'; + return -1; } - case 'u': - case 'U': { - int digits = input[i] == 'u' ? 4 : 8; - unsigned int value = 0; - - i++; - for (int digit = 0; digit < digits; digit++) { - if (!isxdigit(input[i])) { - output[j] = '\0'; - return -1; - } - value = (value << 4) + hex_digit_value(input[i++]); - } - - /* C99 6.4.3 excludes surrogate code points, values above the - * Unicode range, and basic-source characters other than $, @, and - * `. The latter must be spelt directly in source. - */ - if (value > 0x10ffff || (value >= 0xd800 && value <= 0xdfff) || - (value < 0xa0 && value != '$' && value != '@' && - value != '`')) { - output[j] = '\0'; - return -1; - } + if (is_ucn) { j = append_utf8(output, j, output_size, value); if (j < 0) { output[0] = '\0'; return -1; } - break; - } - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': { - /* Octal escape sequence: \ooo (up to 3 digits) */ - int value = 0; - int digit_count = 0; - - while (input[i] >= '0' && input[i] <= '7' && digit_count < 3) { - value = value * 8 + (input[i] - '0'); - i++; - digit_count++; - } - + } else output[j++] = (char) value; - break; - } - default: - /* Unknown escape sequence - treat as literal character */ - output[j++] = input[i++]; - break; - } } output[j] = '\0'; - - /* Check if we ran out of output space */ if (input[i] != '\0') return -1; - return j; } @@ -1006,99 +1291,11 @@ int decode_wstring_units(const char *text, int *units, int capacity) continue; } in++; - switch (text[in]) { - case 'a': - value = '\a'; - in++; - break; - case 'b': - value = '\b'; - in++; - break; - case 'f': - value = '\f'; - in++; - break; - case 'n': - value = '\n'; - in++; - break; - case 'r': - value = '\r'; - in++; - break; - case 't': - value = '\t'; - in++; - break; - case 'v': - value = '\v'; - in++; - break; - case '\\': - value = '\\'; - in++; - break; - case '\'': - value = '\''; - in++; - break; - case '"': - value = '"'; - in++; - break; - case '?': - value = '?'; - in++; - break; - case 'e': - if (strict_c99) - return -1; - value = 27; - in++; - break; - case 'x': - in++; - if (!isxdigit(text[in])) - return -1; - value = 0; - while (isxdigit(text[in])) { - if (value > 0x07ffffffU) - return -1; - value = (value << 4) + hex_digit_value(text[in++]); - } - if (value > 0x7fffffffU) - return -1; - break; - case 'u': - case 'U': { - int digits = text[in] == 'u' ? 4 : 8; - - value = 0; - in++; - for (int i = 0; i < digits; i++) { - if (!isxdigit(text[in])) - return -1; - if (value > 0x07ffffffU) - return -1; - value = (value << 4) + hex_digit_value(text[in++]); - } - if (value > 0x10ffff || (value >= 0xd800 && value <= 0xdfff) || - (value < 0xa0 && value != '$' && value != '@' && value != '`')) - return -1; - break; - } - default: - if (text[in] < '0' || text[in] > '7') - value = (unsigned char) text[in++]; - else { - value = 0; - for (int i = 0; i < 3 && text[in] >= '0' && text[in] <= '7'; - i++) - value = value * 8 + (text[in++] - '0'); - } - break; - } + + bool is_ucn; + + if (!decode_escape_value(text, &in, true, &value, &is_ucn)) + return -1; units[out++] = (int) value; } return out; @@ -1131,7 +1328,8 @@ bool wide_character_constant(const char *literal, int *value) */ int parse_numeric_constant(const char *buffer) { - int i = 0; + int base = numeric_literal_base(buffer); + int i = base == 16 || base == 2 ? 2 : 0; unsigned int value = 0; while (buffer[i]) { /* The lexer keeps C99 integer suffixes in the token. Their type is @@ -1139,36 +1337,18 @@ int parse_numeric_constant(const char *buffer) */ if ((buffer[i] | 32) == 'u' || (buffer[i] | 32) == 'l') break; - if (i == 1 && (buffer[i] | 32) == 'x') { /* hexadecimal */ - value = 0; - i = 2; - while (buffer[i] && (buffer[i] | 32) != 'u' && - (buffer[i] | 32) != 'l') { - char c = buffer[i++]; - value <<= 4; - if (isdigit(c)) - value += c - '0'; - c |= 32; /* convert to lower case */ - if (c >= 'a' && c <= 'f') - value += (c - 'a') + 10; - } - return value; - } - if (i == 1 && (buffer[i] | 32) == 'b') { /* binary */ - value = 0; - i = 2; - while (buffer[i] && (buffer[i] | 32) != 'u' && - (buffer[i] | 32) != 'l') { - char c = buffer[i++]; - value <<= 1; - value += (c == '1'); - } - return value; - } - if (buffer[0] == '0') /* octal */ - value = value * 8 + buffer[i++] - '0'; - else - value = value * 10 + buffer[i++] - '0'; + char c = buffer[i++]; + value *= base; + if (base == 2) + value += c == '1'; + else if (base == 16) { + c |= 32; + if (isdigit(c)) + value += c - '0'; + else if (c >= 'a' && c <= 'f') + value += c - 'a' + 10; + } else + value += c - '0'; } return value; } @@ -1191,6 +1371,7 @@ void type_ensure_fields(type_t *type) return; type->fields = arena_calloc(GENERAL_ARENA, MAX_FIELDS, sizeof(var_t)); + stats_field_slot_count += MAX_FIELDS; /* The field variables come out of a zeroed allocation, so give their * interned name pointers the empty string a reader can dereference. @@ -1235,28 +1416,9 @@ type_t *add_named_type(char *name) return type; } -void add_constant(char alias[], int value) -{ - constant_t *constant = arena_alloc_constant(); - if (!constant) { - printf("Failed to allocate constant_t\n"); - return; - } - - /* Use interned string for constant name */ - strcpy(constant->alias, intern_string(alias)); - constant->value = value; - constant->next = NULL; - hashmap_put(CONSTANTS_MAP, alias, constant); -} - -constant_t *find_constant(char alias[]) -{ - return hashmap_get(CONSTANTS_MAP, alias); -} - /* Enum names declared in a block shadow enclosing names, but must disappear - * with that block. Keep their bindings on the parser's existing block tree. + * with that block. File-scope names use the active translation-unit root of the + * same scope tree. */ void add_scoped_constant(block_t *block, char alias[], int value) { @@ -1268,49 +1430,77 @@ void add_scoped_constant(block_t *block, char alias[], int value) constant->value = value; constant->next = block->constants; block->constants = constant; + if (block == CURRENT_TU_SCOPE) + register_tu_ordinary_name(alias, ORDINARY_CONSTANT, constant); } -/* The kinds of binding an ordinary identifier can have in one scope. Objects, - * functions, typedef names and enumeration constants share C99's ordinary - * identifier name space (6.2.3p1), so each scope binds a name to at most one of - * them. The values are bits, so a lookup can ask for several kinds at once. +/* A file-scope enumeration constant has translation-unit scope, not process + * scope. Keep the parser API but store it on the active input's root. */ -typedef enum { - ORDINARY_NONE = 0, - ORDINARY_CONSTANT = 1, /* enumeration constant declared in the block */ - ORDINARY_VARIABLE = 2, /* local object, or block-scope function alias */ - ORDINARY_TYPEDEF = 4, /* typedef name declared in the block */ - ORDINARY_PARAMETER = 8, /* parameter, seen from the function body's block */ - ORDINARY_ANY = 15 -} ordinary_kind_t; +void add_constant(char alias[], int value) +{ + add_scoped_constant(CURRENT_TU_SCOPE, alias, value); +} -/* The first local of @block at or after index *@pos named @name, or NULL, with - * *@pos left just past it so that a caller can continue to the next one. - * - * A block's locals include all of its IR temporaries, and every declaration - * scans its block's locals at least twice, so the scan is quadratic in the size - * of a block and its compare is what a large one pays for. Settle the first two - * bytes before the library call: temporaries differ in the first, and names - * sharing a prefix letter, such as a run of generated declarations, usually in - * the second. A nonzero first byte that matches means neither name has ended. +/* Keep names declared at file scope in the active unit's ordinary namespace. + * Global storage is merged for code generation, but this small per-unit index + * preserves scope-sensitive collisions with no-linkage enumerators. */ -var_t *find_block_local(block_t *block, const char *name, int *pos) +void add_tu_global_var(var_t *var) { - char head = name[0]; - char second = head ? name[1] : 0; + if (lookup_tu_ordinary_name(file_scope_var_name(var), ORDINARY_VARIABLE, + NULL)) + return; + register_tu_ordinary_name(file_scope_var_name(var), ORDINARY_VARIABLE, var); +} - for (int i = *pos; i < block->locals.size; i++) { - var_t *var = block->locals.elements[i]; - const char *var_name = var->var_name; +static var_t *find_var_in_list(var_list_t *vars, const char *name, int *pos) +{ + for (int i = pos ? *pos : 0; i < vars->size; i++) { + var_t *var = vars->elements[i]; - if (var_name[0] != head || (head && var_name[1] != second)) - continue; - if (!strcmp(var_name, name)) { - *pos = i + 1; + if (identifier_matches(var->var_name, name)) { + if (pos) + *pos = i + 1; return var; } } - *pos = block->locals.size; + if (pos) + *pos = vars->size; + return NULL; +} + +/* Resume at *@pos so callers can continue after a previous match. */ +var_t *find_block_local(block_t *block, const char *name, int *pos) +{ + if (block == CURRENT_TU_SCOPE) { + void *binding = NULL; + + if ((*pos)++) + return NULL; + if (lookup_tu_ordinary_name(name, ORDINARY_VARIABLE, &binding)) + return binding; + return NULL; + } + return find_var_in_list(&block->locals, name, pos); +} + +static var_t *find_tu_declaration(const char *name) +{ + void *binding = NULL; + + if (lookup_tu_ordinary_name(name, ORDINARY_VARIABLE, &binding)) + return binding; + return NULL; +} + +static var_t *find_func_param(func_t *func, const char *name) +{ + if (!func) + return NULL; + for (int i = 0; i < func->num_params; i++) + if (identifier_matches(func->param_defs[i].var_name, name)) + return &func->param_defs[i]; return NULL; } @@ -1326,15 +1516,16 @@ ordinary_kind_t find_block_ordinary(block_t *block, int kinds, void **binding) { - char head = name[0]; void *unused; if (!binding) binding = &unused; + if (block == CURRENT_TU_SCOPE) + return lookup_tu_ordinary_name(name, kinds, binding); if (kinds & ORDINARY_CONSTANT) { for (constant_t *constant = block->constants; constant; constant = constant->next) { - if (!strcmp(constant->alias, name)) { + if (identifier_matches(constant->alias, name)) { *binding = constant; return ORDINARY_CONSTANT; } @@ -1349,27 +1540,49 @@ ordinary_kind_t find_block_ordinary(block_t *block, } if (kinds & ORDINARY_TYPEDEF) { for (typedef_binding_t *td = block->typedefs; td; td = td->next) { - if (td->name[0] == head && !strcmp(td->name, name)) { + if (identifier_matches(td->name, name)) { *binding = td; return ORDINARY_TYPEDEF; } } } if ((kinds & ORDINARY_PARAMETER) && !block->parent && block->func) { - func_t *func = block->func; + var_t *param = find_func_param(block->func, name); - for (int i = 0; i < func->num_params; i++) { - var_t *param = &func->param_defs[i]; - - if (param->var_name[0] == head && !strcmp(param->var_name, name)) { - *binding = param; - return ORDINARY_PARAMETER; - } + if (param) { + *binding = param; + return ORDINARY_PARAMETER; } } return ORDINARY_NONE; } +/* File-scope typedef aliases still use the legacy process-wide type table for + * lookup, but their declaration names also belong in the active unit's + * ordinary-identifier index for same-unit collision checks. + */ +void add_tu_typedef(type_t *type) +{ + const char *name = type->type_name; + + if (!name[0]) + return; + ordinary_kind_t kind = + find_block_ordinary(CURRENT_TU_SCOPE, name, ORDINARY_ANY, NULL); + if (kind == ORDINARY_TYPEDEF) + return; + if (kind != ORDINARY_NONE) + error_at("typedef name conflicts with an ordinary identifier", + cur_token_loc()); + + typedef_binding_t *binding = arena_alloc(BLOCK_ARENA, sizeof(*binding)); + binding->name = intern_string(type->type_name); + binding->type = type; + binding->next = CURRENT_TU_SCOPE->typedefs; + CURRENT_TU_SCOPE->typedefs = binding; + register_tu_ordinary_name(name, ORDINARY_TYPEDEF, binding); +} + /* The enumeration constant @alias as seen from @block, or NULL when none is * visible. Constants share the ordinary identifier name space, so an object, * parameter or typedef name declared in a nearer scope hides an outer constant @@ -1388,11 +1601,13 @@ constant_t *find_scoped_constant(char alias[], block_t *block) for (owner = block; owner && owner != GLOBAL_BLOCK; owner = owner->parent) if (find_block_ordinary(owner, alias, ORDINARY_CONSTANT, &constant)) break; - if (!owner || owner == GLOBAL_BLOCK) - constant = find_constant(alias); - if (!constant) - return NULL; - for (; block != owner; block = block->parent) { + if (!owner || owner == GLOBAL_BLOCK) { + /* Function scopes are not parented to the translation-unit scope. */ + owner = GLOBAL_BLOCK; + if (!lookup_tu_ordinary_name(alias, ORDINARY_CONSTANT, &constant)) + return NULL; + } + for (; block && block != owner; block = block->parent) { if (find_block_ordinary( block, alias, ORDINARY_VARIABLE | ORDINARY_TYPEDEF | ORDINARY_PARAMETER, @@ -1417,7 +1632,7 @@ void add_type_tag(block_t *block, char name[], type_t *type) type_t *find_local_type_tag(char name[], block_t *block) { for (type_tag_t *tag = block->type_tags; tag; tag = tag->next) - if (!strcmp(tag->name, name)) + if (identifier_matches(tag->name, name)) return tag->type; return NULL; } @@ -1427,6 +1642,15 @@ type_t *find_local_type_tag(char name[], block_t *block) */ #define ENUM_TAG_KIND TYPE_int +/* File-scope tags live in the active translation unit, not the merged block + * that owns global object declarations. Some parser paths still pass that + * merged block to type lookup; normalize those file-scope arguments here. + */ +static block_t *tag_scope(block_t *block) +{ + return !block || block == GLOBAL_BLOCK ? CURRENT_TU_SCOPE : block; +} + /* C99 6.7.2.3p3: every declaration of a tag names the same kind of type, and * struct, union and enum tags share one name space, so a tag found under * another keyword is an error rather than a miss. @kind is TYPE_struct, @@ -1442,19 +1666,20 @@ static type_t *check_tag_kind(type_t *type, base_type_t kind) /* The tag @name of @kind as seen from @block, or NULL when no tag of that name * is visible. Tags are registered with the block that declares them, file-scope - * ones with GLOBAL_BLOCK, which a function body's chain does not reach on its - * own, so a NULL @block sees file scope only. A tag declared in some other - * block is never found. Looking in the tag table, not the type table, also - * keeps a typedef name out of it. + * ones with CURRENT_TU_SCOPE, which a function body's chain does not reach on + * its own, so a NULL @block sees the current file scope only. A tag declared in + * another translation unit is never found. Looking in the tag table, not the + * type table, also keeps a typedef name out of it. */ static type_t *find_visible_tag(char name[], block_t *block, base_type_t kind) { type_t *type = NULL; - for (; block && !type; block = block->parent) + block = tag_scope(block); + for (; block && block != CURRENT_TU_SCOPE && !type; block = block->parent) type = find_local_type_tag(name, block); if (!type) - type = find_local_type_tag(name, GLOBAL_BLOCK); + type = find_local_type_tag(name, CURRENT_TU_SCOPE); return check_tag_kind(type, kind); } @@ -1483,10 +1708,10 @@ type_t *find_record_tag(char name[], block_t *block, base_type_t kind) */ type_t *reference_record_tag(char name[], block_t *block, base_type_t kind) { + block = tag_scope(block); type_t *type = find_record_tag(name, block, kind); - return type ? type - : declare_record_tag(name, block ? block : GLOBAL_BLOCK, kind); + return type ? type : declare_record_tag(name, block, kind); } /* The tag @name that @block itself declares, created incomplete when it has @@ -1495,6 +1720,7 @@ type_t *reference_record_tag(char name[], block_t *block, base_type_t kind) */ type_t *local_record_tag(char name[], block_t *block, base_type_t kind) { + block = tag_scope(block); type_t *type = check_tag_kind(find_local_type_tag(name, block), kind); return type ? type : declare_record_tag(name, block, kind); @@ -1537,6 +1763,7 @@ type_t *reference_enum_tag(char name[], block_t *block) /* The enum tag @name that @block itself declares, or NULL. */ type_t *local_enum_tag(char name[], block_t *block) { + block = tag_scope(block); return check_tag_kind(find_local_type_tag(name, block), ENUM_TAG_KIND); } @@ -1570,6 +1797,11 @@ void add_block_typedef(block_t *block, char name[], type_t *type) */ type_t *find_visible_type(const char *name, block_t *block) { + bool searched_tu = false; + + if (!block || block == GLOBAL_BLOCK) + block = CURRENT_TU_SCOPE; + /* Parameters are tried with the function body's outermost block, so they * too hide a file-scope typedef unless a nearer binding has hidden them. */ @@ -1578,12 +1810,24 @@ type_t *find_visible_type(const char *name, block_t *block) ordinary_kind_t kind = find_block_ordinary(block, name, ORDINARY_ANY, &binding); + if (block == CURRENT_TU_SCOPE) + searched_tu = true; + if (kind == ORDINARY_TYPEDEF) + return resolve_type_alias(((typedef_binding_t *) binding)->type); + if (kind != ORDINARY_NONE) + return NULL; + } + if (!searched_tu) { + void *binding; + ordinary_kind_t kind = + find_block_ordinary(CURRENT_TU_SCOPE, name, ORDINARY_ANY, &binding); + if (kind == ORDINARY_TYPEDEF) - return ((typedef_binding_t *) binding)->type; + return resolve_type_alias(((typedef_binding_t *) binding)->type); if (kind != ORDINARY_NONE) return NULL; } - return find_type(name, 1); + return find_builtin_type(name); } var_t *find_member(const char token[], type_t *type) @@ -1598,66 +1842,35 @@ var_t *find_member(const char token[], type_t *type) if (!type) return NULL; - char head = token[0]; - for (int i = 0; i < type->num_fields; i++) { - if (type->fields[i].var_name[0] != head) - continue; - if (!strcmp(type->fields[i].var_name, token)) + if (identifier_matches(type->fields[i].var_name, token)) return &type->fields[i]; } return NULL; } -/* Name lookup is the parser's inner loop: every identifier walks the enclosing - * scopes, then the parameter list, then the globals, and all but the one match - * is a strcmp against a name that differs immediately. A call into the C - * library's vectorized strcmp costs far more than the comparison it performs on - * such names, so each scan settles the common case -- a different first letter - * -- before making the call. Names are never empty, so reading the first byte - * of either side is always in bounds. - */ +/* Search locals from the innermost scope outward, then parameters. */ var_t *find_local_var(const char *token, block_t *block) { func_t *func = block->func; - char head = token[0]; + var_t *var; for (; block; block = block->parent) { - var_list_t *var_list = &block->locals; - for (int i = 0; i < var_list->size; i++) { - var_t *var = var_list->elements[i]; - if (var->var_name[0] != head) - continue; - if (!strcmp(var->var_name, token)) - return var; - } + if ((var = find_var_in_list(&block->locals, token, NULL))) + return var; } - if (func) { - for (int i = 0; i < func->num_params; i++) { - var_t *param = &func->param_defs[i]; - if (param->var_name[0] != head) - continue; - if (!strcmp(param->var_name, token)) - return param; - } - } - return NULL; + return find_func_param(func, token); } var_t *find_global_var(const char *token) { - var_list_t *var_list = &GLOBAL_BLOCK->locals; - char head = token[0]; + var_t *unit_var = find_tu_declaration(token); - for (int i = 0; i < var_list->size; i++) { - var_t *var = var_list->elements[i]; - if (var->var_name[0] != head) - continue; - if (!strcmp(var->var_name, token)) - return var; - } - return NULL; + if (unit_var && !unit_var->is_block_scope_function_declaration) + return find_func(unit_var->var_name) ? NULL : unit_var; + + return find_var_in_list(&GLOBAL_BLOCK->locals, token, NULL); } var_t *find_var(char *token, block_t *parent) @@ -1705,17 +1918,18 @@ int size_var(var_t *var) */ func_t *add_func(char *func_name, bool synthesize) { + char *stable_name; func_t *func = hashmap_get(FUNC_MAP, func_name); if (func) return func; + stable_name = intern_string(func_name); func = arena_alloc_func(); - hashmap_put(FUNC_MAP, func_name, func); + hashmap_put_borrowed(FUNC_MAP, stable_name, func); for (int i = 0; i < MAX_PARAMS; i++) func->param_defs[i].var_name = ""; - /* Use interned string for function name */ - func->return_def.var_name = intern_string(func_name); + func->return_def.var_name = stable_name; /* Prepare space for function arguments. * @@ -1771,6 +1985,22 @@ func_t *find_func(char *func_name) return hashmap_get(FUNC_MAP, func_name); } +int function_entry_address(char *name) +{ + func_t *func = find_func(name); + + if (func->bbs) + return elf_code_start + func->bbs->elf_offset; + if (dynlink) + return dynamic_sections.elf_plt_start + func->plt_offset; + + /* compile_native_vir() rejects a used, undefined function first. */ + char message[MAX_LINE_LEN]; + snprintf(message, MAX_LINE_LEN, "call to undefined function '%s'", name); + fatal(message); + return 0; +} + /* Create a basic block and set the scope of variables to 'parent' block */ basic_block_t *bb_create(block_t *parent) { @@ -1778,7 +2008,7 @@ basic_block_t *bb_create(block_t *parent) * zeroing (live_gen, live_kill, live_in, live_out, DF, RDF, dom_next, etc.) * This is simpler and safer than manually initializing everything. */ - basic_block_t *bb = arena_calloc(BB_ARENA, 1, sizeof(basic_block_t)); + basic_block_t *bb = arena_alloc_bb(); /* Initialize non-zero fields */ bb->scope = parent; @@ -1790,28 +2020,9 @@ basic_block_t *bb_create(block_t *parent) */ bb->elf_offset = -1; - if (dump_ir || dump_dot) { - /* MAX_VAR_LEN spent 128 bytes on a string that is always ".label." plus - * an int. A self-compile calls bb_create() 52k times, so that was 6.4 - * MiB of arena where 1.2 MiB does. - */ - bb->bb_label_name = arena_alloc(GENERAL_ARENA, MAX_LABEL_LEN); - snprintf(bb->bb_label_name, MAX_LABEL_LEN, ".label.%d", bb_label_idx++); - } - return bb; } -/* Bumped once per compute_live_in() call; a variable belongs to the set being - * built when its stamp equals the current value. - */ -int liveness_gen; - -/* Bumped once per recompute_live_out() call, stamping the union of successor - * live_in sets as it is assembled. - */ -int live_merge_gen; - /* log2 of @v when it is a power of two, otherwise -1. */ int exact_log2(int v) { @@ -1839,67 +2050,21 @@ void *arena_grow(arena_t *arena, int limit, char *what) { - int new_cap = *cap ? *cap << 1 : first; + int new_cap; + + if (*cap < 0 || elem_sz <= 0 || first <= 0 || *cap > INT_MAX / 2) + return NULL; + new_cap = *cap ? *cap << 1 : first; + if (new_cap > INT_MAX / elem_sz) + return NULL; if (limit && new_cap > limit) limit_error(what); void *grown = arena_realloc(arena, ptr, *cap * elem_sz, new_cap * elem_sz); - *cap = new_cap; + if (grown) + *cap = new_cap; return grown; } -/* Record another SSA version of @v, growing the version array as needed. */ -void var_add_subscript(var_t *v, var_t *sub) -{ - if (v->subscripts_idx >= v->subscripts_cap) - v->subscripts = - arena_grow(BLOCK_ARENA, (char *) v->subscripts, &v->subscripts_cap, - sizeof(var_t *), 4, 0, NULL); - v->subscripts[v->subscripts_idx++] = sub; -} - -/* The first SSA version of @v, or NULL when it has none. Callers relied on the - * old array being zero-filled to get NULL here. - */ -var_t *var_subscript0(var_t *v) -{ - if (!v->subscripts_idx) - return NULL; - return v->subscripts[0]; -} - -/* Detach @v from any version array it inherited from a by-value copy. */ -void var_reset_subscripts(var_t *v) -{ - /* memcpy'd from a base, so the copy would otherwise alias the base's - * renaming state as well as its version list. - */ - v->rename = NULL; - v->subscripts = NULL; - v->subscripts_idx = 0; - v->subscripts_cap = 0; -} - -/* Append to a basic block's dominance frontier, growing the array as needed. - * The array starts unallocated, so a block that never contributes to a frontier - * costs nothing beyond the pointer. - */ -void bb_add_df(basic_block_t *bb, basic_block_t *df) -{ - if (bb->df_idx >= bb->df_cap) - bb->DF = arena_grow(BB_ARENA, (char *) bb->DF, &bb->df_cap, - sizeof(basic_block_t *), 8, 0, NULL); - bb->DF[bb->df_idx++] = df; -} - -/* The reverse-dominance-frontier counterpart of bb_add_df(). */ -void bb_add_rdf(basic_block_t *bb, basic_block_t *rdf) -{ - if (bb->rdf_idx >= bb->rdf_cap) - bb->RDF = arena_grow(BB_ARENA, (char *) bb->RDF, &bb->rdf_cap, - sizeof(basic_block_t *), 8, 0, NULL); - bb->RDF[bb->rdf_idx++] = rdf; -} - /* The pred-succ pair must have only one connection */ void bb_connect(basic_block_t *pred, basic_block_t *succ, @@ -1942,79 +2107,76 @@ void bb_connect(basic_block_t *pred, default: abort(); } + vir_frontend_note_edge(pred, succ); } -/* The pred-succ pair must have only one connection */ -void bb_disconnect(basic_block_t *pred, basic_block_t *succ) -{ - for (int i = 0; i < succ->prev_idx; i++) { - if (succ->prev[i].bb == pred) { - switch (succ->prev[i].type) { - case NEXT: - pred->next = NULL; - break; - case THEN: - pred->then_ = NULL; - break; - case ELSE: - pred->else_ = NULL; - break; - default: - abort(); - } - - succ->prev[i].bb = NULL; - break; - } - } -} - -/* Count the predecessors still wired to 'bb'. bb_disconnect() leaves holes in - * prev[], so prev_idx is only a high-water mark and the entries must be counted - * rather than trusted. - */ -int bb_pred_count(const basic_block_t *bb) +/* bb_disconnect() leaves holes in prev[], so prev_idx alone is not enough. */ +bool bb_has_pred(const basic_block_t *bb) { - int n = 0; - - for (int i = 0; i < bb->prev_idx; i++) { + for (int i = 0; i < bb->prev_idx; i++) if (bb->prev[i].bb) - n++; - } - return n; + return true; + return false; } -/* The symbol is an argument of function or the variable in declaration */ -void add_symbol(basic_block_t *bb, var_t *var) +/* Return the sole live predecessor, or NULL when the block has zero or many. */ +basic_block_t *bb_sole_pred(const basic_block_t *bb) { - if (!bb) - return; - symbol_t *sym; - for (sym = bb->symbol_list.head; sym; sym = sym->next) { - if (sym->var == var) - return; - } + basic_block_t *pred = NULL; - sym = arena_alloc_symbol(); - sym->var = var; + for (int i = 0; i < bb->prev_idx; i++) { + basic_block_t *candidate = bb->prev[i].bb; - if (!bb->symbol_list.head) { - sym->index = 0; - bb->symbol_list.head = sym; - bb->symbol_list.tail = sym; - } else { - sym->index = bb->symbol_list.tail->index + 1; - bb->symbol_list.tail->next = sym; - bb->symbol_list.tail = sym; + if (!candidate) + continue; + if (pred) + return NULL; + pred = candidate; } + return pred; } -/* Whether @var is a volatile object named by its declaration, as opposed to a - * temporary the parser generated. Temporaries are named ".tN". +/* Pointer-star qualifiers describe pointer storage independently of its base. */ +static int var_volatile_pointer_depth(const var_t *var) +{ + return var->ptr_level + var->type->ptr_level + var->is_func + + !!(var->array_size || var->has_unsized_array); +} +bool var_volatile_storage(const var_t *var) +{ + if (!var || !var->type) + return false; + int depth = var_volatile_pointer_depth(var); + return depth ? depth <= 32 && + (var->pointer_volatile_mask & (1U << (depth - 1))) + : var->is_volatile || var->type->is_volatile_qualified; +} +bool var_volatile_pointee(const var_t *var) +{ + if (!var || !var->type) + return false; + int depth = var_volatile_pointer_depth(var); + const func_t *signature = var->pointee_func_signature; + if (!signature && !var->is_func && var->type->func_signature && + !var->type->is_direct_function_type) + signature = var->type->func_signature; + if (signature) { + const var_t *result = &signature->return_def; + int return_depth = + var->type->func_signature && !var->type->is_direct_function_type + ? 0 + : result->ptr_level + result->type->ptr_level; + if (depth == return_depth + 1) + return var->callback_is_volatile; + } + return depth > 1 ? depth <= 33 && + (var->pointer_volatile_mask & (1U << (depth - 2))) + : var->is_volatile || var->type->is_volatile_qualified; +} bool var_is_volatile_object(const var_t *var) { - return var && var->is_volatile && var->var_name[0] != '.'; + return var_volatile_storage(var) && var->var_name[0] != '.'; } /* A volatile object a primary expression has named and no instruction has read @@ -2036,26 +2198,6 @@ void add_insn(block_t *block, bb->scope = block; - insn_t *n = arena_alloc(INSN_ARENA, sizeof(insn_t)); - n->next = NULL; - n->prev = NULL; - n->opcode = op; - n->rd = rd; - n->rs1 = rs1; - n->rs2 = rs2; - - /* Only a select names a third source. The allocation is not zeroed and - * every field is set here by hand, so this one has to be too. - */ - n->rs3 = NULL; - n->sz = sz; - n->useful = false; - n->belong_to = bb; - n->phi_ops = NULL; - n->idx = 0; - - n->str = str ? intern_string(str) : NULL; - /* An instruction reading the object discard_operand() watches has given it * the read it is owed. */ @@ -2067,6 +2209,8 @@ void add_insn(block_t *block, * optimization */ if ((op == OP_address_of || op == OP_global_address_of) && rs1) { + if (rd) + rd->is_volatile_access = var_volatile_storage(rs1); rs1->address_taken = true; rs1->is_const = false; /* disable constant optimization */ } @@ -2076,16 +2220,85 @@ void add_insn(block_t *block, * its slot as though its address had escaped, so that each access by name * reaches memory rather than a register copy. */ - if (op == OP_allocat && rd && rd->is_volatile && !rd->is_global) + if (op == OP_allocat && var_volatile_storage(rd) && !rd->is_global) rd->address_taken = true; - if (!bb->insn_list.head) - bb->insn_list.head = n; - else - bb->insn_list.tail->next = n; + vir_frontend_note_insn(bb, op, rd, rs1, rs2, sz, + str ? intern_string(str) : NULL); +} + +void dump_stats_phase(const char *phase) +{ + if (!dump_stats) + return; + + /* Keep each call within the compiler's current fixed-argument ABI limit. + * The line grouping is also intentionally stable for benchmark parsers. + */ + fprintf(stderr, "stats phase=%s funcs=%d vars=%d bbs=%d\n", phase, + stats_func_count, stats_var_count, stats_bb_count); + if (!strcmp(phase, "parse")) { + fprintf(stderr, "stats phase=parse translation_units=%d\n", + translation_unit_count); + fprintf(stderr, + "stats phase=parse tu_index_lookups=%d " + "tu_index_registrations=%d\n", + stats_tu_index_lookups, stats_tu_index_registrations); + } + fprintf(stderr, + "stats phase=%s field_slots=%d param_slots=%d var_slots=%d\n", + phase, stats_field_slot_count, stats_func_count * MAX_PARAMS, + stats_var_count + stats_field_slot_count + + stats_func_count * MAX_PARAMS); + vir_stats_t totals = {0}; + for (func_t *func = FUNC_LIST.head; func; func = func->next) { + const vir_function_t *graph = vir_frontend_function(func); + if (!graph) + continue; + vir_stats_t stats; + vir_collect_stats(graph, &stats); + totals.blocks += stats.blocks; + totals.values += stats.values; + totals.params += stats.params; + totals.effects += stats.effects; + totals.arena_capacity += stats.arena_capacity; + } + fprintf(stderr, + "stats phase=%s vir_blocks=%d vir_values=%d vir_params=%d\n", phase, + totals.blocks, totals.values, totals.params); + fprintf(stderr, "stats phase=%s vir_effects=%d vir_arena=%d mop=%d\n", + phase, totals.effects, totals.arena_capacity, stats_mop_count); + fprintf(stderr, "stats phase=%s arena_capacity block=%d bb=%d\n", phase, + BLOCK_ARENA ? BLOCK_ARENA->total_bytes : 0, + BB_ARENA ? BB_ARENA->total_bytes : 0); + fprintf(stderr, "stats phase=%s arena_capacity token=%d general=%d\n", + phase, TOKEN_ARENA ? TOKEN_ARENA->total_bytes : 0, + GENERAL_ARENA ? GENERAL_ARENA->total_bytes : 0); + fprintf(stderr, "stats phase=%s arena_capacity_peak block=%d bb=%d\n", + phase, BLOCK_ARENA ? BLOCK_ARENA->peak_bytes : 0, + BB_ARENA ? BB_ARENA->peak_bytes : 0); + + /* The preprocessor's arena is retired once the parser's stream is copied + * out of it, so the high-water mark is the larger of the two. + */ + int token_peak = stats_retired_token_arena_peak; + if (TOKEN_ARENA && TOKEN_ARENA->peak_bytes > token_peak) + token_peak = TOKEN_ARENA->peak_bytes; + fprintf(stderr, "stats phase=%s arena_capacity_peak token=%d general=%d\n", + phase, token_peak, GENERAL_ARENA ? GENERAL_ARENA->peak_bytes : 0); + fprintf(stderr, "stats phase=%s arena_total_capacity=%d\n", phase, + stats_live_arena_capacity); + fprintf(stderr, "stats phase=%s arena_total_capacity_peak=%d\n", phase, + stats_peak_arena_capacity); +} + +void dump_stats_layout(void) +{ + if (!dump_stats) + return; - n->prev = bb->insn_list.tail; - bb->insn_list.tail = n; + fprintf(stderr, "stats layout var=%d bb=%d mop=%d\n", (int) sizeof(var_t), + (int) sizeof(basic_block_t), (int) sizeof(vir_mop_t)); } strbuf_t *strbuf_create(int init_capacity) @@ -2096,7 +2309,7 @@ strbuf_t *strbuf_create(int init_capacity) array->size = 0; array->capacity = init_capacity; - array->plain_source = false; + array->plain_end = 0; array->elements = malloc(array->capacity * sizeof(char)); if (!array->elements) { free(array); @@ -2158,7 +2371,7 @@ bool strbuf_puts(strbuf_t *src, const char *value) return false; } - strncpy(src->elements + src->size, value, len); + memcpy(src->elements + src->size, value, len); src->size += len; return true; @@ -2178,23 +2391,22 @@ void strbuf_free(strbuf_t *src) */ void global_init(void) { + internal_linkage_name_idx = 0; FUNC_LIST.head = NULL; FUNC_LIST.tail = NULL; - memset(REGS, 0, sizeof(regfile_t) * REG_CNT); /* Initialize arenas first so we can use them for allocation */ BLOCK_ARENA = arena_init(DEFAULT_ARENA_SIZE); /* Variables/blocks */ - INSN_ARENA = arena_init(LARGE_ARENA_SIZE); /* Instructions - high usage */ - BB_ARENA = arena_init(SMALL_ARENA_SIZE); /* Basic blocks - low usage */ - HASHMAP_ARENA = arena_init(DEFAULT_ARENA_SIZE); /* Hash nodes */ + BB_ARENA = arena_init(SMALL_ARENA_SIZE); /* Basic blocks - low usage */ TOKEN_ARENA = arena_init(LARGE_ARENA_SIZE); GENERAL_ARENA = - arena_init(DEFAULT_ARENA_SIZE); /* For TYPES and PH2_IR_FLATTEN */ + arena_init(DEFAULT_ARENA_SIZE); /* For TYPES and flattened IR */ /* Use arena allocation for better memory management */ TYPES = arena_calloc(GENERAL_ARENA, MAX_TYPES, sizeof(type_t)); - PH2_IR_FLATTEN = - arena_alloc(GENERAL_ARENA, MAX_IR_INSTR * sizeof(ph2_ir_t *)); + MOP_FLAT = NULL; + mop_flat_capacity = 0; + mop_flat_count = 0; /* Initialize string pool for identifier deduplication */ string_pool = arena_alloc(GENERAL_ARENA, sizeof(string_pool_t)); @@ -2208,7 +2420,6 @@ void global_init(void) TOKEN_CACHE = hashmap_create(DEFAULT_SRC_FILE_COUNT); SRC_FILE_MAP = hashmap_create(DEFAULT_SRC_FILE_COUNT); FUNC_MAP = hashmap_create(DEFAULT_FUNCS_SIZE); - CONSTANTS_MAP = hashmap_create(MAX_CONSTANTS); LIBC_SRC = strbuf_create(4096); elf_code = strbuf_create(MAX_CODE); @@ -2248,6 +2459,23 @@ void global_init(void) /* Forward declaration for lexer cleanup */ void lexer_cleanup(void); +/* Free every block of @arena older than @keep, returning the bytes freed. */ +static int arena_free_blocks_after(arena_t *arena, arena_block_t *keep) +{ + int freed = 0; + + while (keep->next) { + arena_block_t *done = keep->next; + + keep->next = done->next; + freed += done->capacity; + arena->total_bytes -= done->capacity; + stats_live_arena_capacity -= done->capacity; + arena_block_free(done); + } + return freed; +} + /* Free empty trailing blocks from an arena safely. This only frees blocks that * come after the last used block, ensuring no pointers are invalidated. * @@ -2281,42 +2509,74 @@ int arena_free_trailing_blocks(arena_t *arena) if (!last_used) last_used = arena->head; - /* Free all blocks after last_used */ - int freed = 0; - if (last_used->next) { - block = last_used->next; - last_used->next = NULL; - - while (block) { - arena_block_t *next = block->next; - freed += block->capacity; - arena->total_bytes -= block->capacity; - arena_block_free(block); - block = next; - } - } + return arena_free_blocks_after(arena, last_used); +} - return freed; +/* Free @retired, a token arena, keeping its peak for --stats. The lexer maps + * borrow storage from the token arena, so they go first; they rebuild + * themselves in TOKEN_ARENA should anything look a keyword up. + */ +void retire_token_arena(arena_t *retired) +{ + lexer_cleanup(); + if (!retired) + return; + if (retired->peak_bytes > stats_retired_token_arena_peak) + stats_retired_token_arena_peak = retired->peak_bytes; + arena_free(retired); +} + +/* The newest block of the compacted token stream and the end of the stream in + * it. Older blocks hold nothing but stream tokens, in order, so once the parser + * has moved past one at file scope, where no earlier token is held, it is + * garbage. Anything allocated after compaction is newer than both, and tokens + * spliced in during parsing are never in a block that gets freed. + */ +static arena_block_t *token_stream_block; +static char *token_stream_end; + +void mark_token_stream_end(void) +{ + token_stream_block = TOKEN_ARENA->head; + token_stream_end = token_stream_block->memory + token_stream_block->offset; +} + +/* Free the stream blocks the parser has read past. */ +void release_parsed_tokens(void) +{ + arena_block_t *block = token_stream_block; + char *at = (char *) cur_token; + char *end = token_stream_end; + + while (block && !(at >= block->memory && at < end)) { + block = block->next; + end = block ? block->memory + block->offset : NULL; + } + if (block) + arena_free_blocks_after(TOKEN_ARENA, block); } /* Release the whole token arena and the source buffers behind it. * - * Every token, macro, hide set and conditional-inclusion record lives in - * TOKEN_ARENA, and nothing survives parsing: identifiers and string literals - * reach the parser through intern_string(), which copies into GENERAL_ARENA, - * and every parser entry point copies a token's text into a local buffer before - * storing it. So once parse() returns, all 17 MiB of it is garbage that would - * otherwise stay resident through the memory peak in reg_alloc(). + * By now TOKEN_ARENA holds the parsed token stream and whatever constant + * evaluation copied during parsing, and nothing survives parsing: identifiers + * and string literals reach the parser through intern_string(), which copies + * into GENERAL_ARENA, and every parser entry point copies a token's text into a + * local buffer before storing it. So once parse() returns, all of it is garbage + * that would otherwise stay resident through machine lowering. * * The source buffers in SRC_FILE_MAP exist only to quote a line in a parse * error, so they go at the same time. */ void release_token_arena(void) { - if (TOKEN_ARENA) { - arena_free(TOKEN_ARENA); - TOKEN_ARENA = NULL; - } + token_stream_block = NULL; + + /* Parsing is complete, so nothing reads the tokens or the lexer maps again. + * global_release() may call this cleanup again safely. + */ + retire_token_arena(TOKEN_ARENA); + TOKEN_ARENA = NULL; /* Every error_at() site is in the preprocessor or the parser, and both are * done by the time this runs, so no line will be quoted again. LIBC_SRC is @@ -2337,73 +2597,86 @@ void release_token_arena(void) } } -/* Compact all arenas to reduce memory usage after compilation phases. This - * safely frees only trailing empty blocks without invalidating pointers. - * - * Return: Total bytes freed across all arenas. +/* The parser's basic blocks only carry a function to its VIR graph, and the + * last graph, the global one, is finished before lowering starts. Lowering + * gives every function with a graph blocks of its own, or none when it is + * unused, so the parser's blocks go now. A function with no graph, such as the + * synthesized __syscall, keeps an empty block as its call target. */ -int compact_all_arenas(void) +void release_parse_blocks(void) { - int total_saved = 0; + arena_t *parsed = BB_ARENA; - /* Free trailing blocks from each arena */ - total_saved += arena_free_trailing_blocks(BLOCK_ARENA); - total_saved += arena_free_trailing_blocks(INSN_ARENA); - total_saved += arena_free_trailing_blocks(BB_ARENA); - total_saved += arena_free_trailing_blocks(HASHMAP_ARENA); - total_saved += arena_free_trailing_blocks(GENERAL_ARENA); + BB_ARENA = arena_init(SMALL_ARENA_SIZE); + for (int i = 0; i < FUNC_MAP->cap; i++) { + func_t *func = + FUNC_MAP->table[i].occupied ? FUNC_MAP->table[i].val : NULL; + basic_block_t *bb; - return total_saved; + if (!func || !func->bbs) + continue; + if (func->vir_context) { + func->bbs = NULL; + continue; + } + bb = arena_alloc_bb(); + bb->belong_to = func->bbs->belong_to; + bb->elf_offset = func->bbs->elf_offset; + func->bbs = bb; + } + arena_free(parsed); } -/* Compact specific arenas based on compilation phase. Different phases have - * different memory usage patterns. - * - * @phase_mask: Bitmask using COMPACT_ARENA_* defines - * to indicate which arenas to compact. - * - * Return: Total bytes freed. - */ -int compact_arenas_selective(int phase_mask) +void release_block_arena(void) +{ + arena_free(BLOCK_ARENA); + BLOCK_ARENA = NULL; +#ifdef __GLIBC__ + /* main() keeps freed memory around for reuse, but these variables sit in + * the middle of the heap and the backend allocates nothing like them. + * Handing the pages back once costs a little refaulting and lowers the peak + * the ELF writer would otherwise set on top of them. + */ + malloc_trim(0); +#endif +} + +static int compact_arenas(int phase_mask) { int total_saved = 0; if (phase_mask & COMPACT_ARENA_BLOCK) total_saved += arena_free_trailing_blocks(BLOCK_ARENA); - - if (phase_mask & COMPACT_ARENA_INSN) - total_saved += arena_free_trailing_blocks(INSN_ARENA); - if (phase_mask & COMPACT_ARENA_BB) total_saved += arena_free_trailing_blocks(BB_ARENA); - - if (phase_mask & COMPACT_ARENA_HASHMAP) - total_saved += arena_free_trailing_blocks(HASHMAP_ARENA); - if (phase_mask & COMPACT_ARENA_GENERAL) total_saved += arena_free_trailing_blocks(GENERAL_ARENA); return total_saved; } +/* Compact selected arenas for their compilation phase. */ +int compact_arenas_selective(int phase_mask) +{ + return compact_arenas(phase_mask); +} + void global_release(void) { - /* Cleanup lexer hashmaps */ - lexer_cleanup(); + /* Release token and source storage before the remaining compiler state. */ + release_token_arena(); /* Free string interning hashmaps */ if (string_pool && string_pool->strings) hashmap_free(string_pool->strings); if (string_literal_pool && string_literal_pool->literals) hashmap_free(string_literal_pool->literals); + hashmap_free(TU_ORDINARY_MAP); + TU_ORDINARY_MAP = NULL; arena_free(BLOCK_ARENA); - arena_free(INSN_ARENA); arena_free(BB_ARENA); - arena_free(HASHMAP_ARENA); - if (TOKEN_ARENA) - arena_free(TOKEN_ARENA); - arena_free(GENERAL_ARENA); /* free TYPES and PH2_IR_FLATTEN */ + arena_free(GENERAL_ARENA); /* free TYPES */ /* Every value in TOKEN_CACHE is one heap-allocated token_stream_t: the * tokens it spans come from TOKEN_ARENA, which is already gone, but the @@ -2417,31 +2690,34 @@ void global_release(void) } } hashmap_free(TOKEN_CACHE); - hashmap_free(SRC_FILE_MAP); hashmap_free(FUNC_MAP); hashmap_free(INCLUSION_MAP); - hashmap_free(CONSTANTS_MAP); - - strbuf_free(LIBC_SRC); - strbuf_free(elf_code); - strbuf_free(elf_data); - strbuf_free(elf_rodata); - strbuf_free(elf_header); - strbuf_free(elf_program_header); - strbuf_free(elf_symtab); - strbuf_free(elf_strtab); - strbuf_free(elf_shstrtab); - strbuf_free(elf_section_header); - strbuf_free(dynamic_sections.elf_interp); - strbuf_free(dynamic_sections.elf_dynamic); - strbuf_free(dynamic_sections.elf_dynsym); - strbuf_free(dynamic_sections.elf_dynstr); - if (dynamic_sections.use_relaplt) - strbuf_free(dynamic_sections.elf_relaplt); - else - strbuf_free(dynamic_sections.elf_relplt); - strbuf_free(dynamic_sections.elf_plt); - strbuf_free(dynamic_sections.elf_got); + + strbuf_t *buffers[] = { + LIBC_SRC, + elf_code, + elf_data, + elf_rodata, + elf_header, + elf_program_header, + elf_symtab, + elf_strtab, + elf_shstrtab, + elf_section_header, + dynamic_sections.elf_interp, + dynamic_sections.elf_dynamic, + dynamic_sections.elf_dynsym, + dynamic_sections.elf_dynstr, + dynamic_sections.use_relaplt ? dynamic_sections.elf_relaplt + : dynamic_sections.elf_relplt, + dynamic_sections.elf_plt, + dynamic_sections.elf_got, + }; + for (size_t i = 0; i < sizeof(buffers) / sizeof(*buffers); i++) + strbuf_free(buffers[i]); + MOP_FLAT = NULL; + mop_flat_count = 0; + mop_flat_capacity = 0; } /* The function whose body the back half of the pipeline is working on, or NULL @@ -2580,300 +2856,14 @@ __noreturn void error_at(char *msg, source_location_t *loc) exit(1); } -void print_indent(int indent) -{ - for (int i = 0; i < indent; i++) - printf("\t"); -} - -void dump_bb_insn(const func_t *func, - const basic_block_t *bb, - bool *at_func_start) -{ - if (!bb) - return; - const var_t *rd, *rs1, *rs2; - - if (bb != func->bbs && bb->insn_list.head) { - if (!at_func_start[0]) - printf("%s:\n", bb->bb_label_name); - else - at_func_start[0] = false; - } - - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - rd = insn->rd; - rs1 = insn->rs1; - rs2 = insn->rs2; - - switch (insn->opcode) { - case OP_unwound_phi: - /* Ignored */ - continue; - case OP_allocat: - print_indent(1); - printf("allocat %s", rd->type->type_name); - - for (int i = 0; i < rd->ptr_level; i++) - printf("*"); - - printf(" %%%s", rd->var_name); - - if (rd->array_size > 0) - printf("[%d]", rd->array_size); - - break; - case OP_load_constant: - print_indent(1); - printf("const %%%s, %d", rd->var_name, rd->init_val); - break; - case OP_load_data_address: - print_indent(1); - /* offset from .data section */ - printf("%%%s = .data (%d)", rd->var_name, rd->init_val); - break; - case OP_load_rodata_address: - print_indent(1); - /* offset from .rodata section */ - printf("%%%s = .rodata (%d)", rd->var_name, rd->init_val); - break; - case OP_address_of: - print_indent(1); - printf("%%%s = &(%%%s)", rd->var_name, rs1->var_name); - break; - case OP_assign: - print_indent(1); - printf("%%%s = %%%s", rd->var_name, rs1->var_name); - break; - case OP_branch: - print_indent(1); - printf("br %%%s, %s, %s", rs1->var_name, bb->then_->bb_label_name, - bb->else_->bb_label_name); - break; - case OP_jump: - print_indent(1); - printf("jmp %s", bb->next->bb_label_name); - break; - case OP_label: - print_indent(0); - printf("%s:", insn->str); - break; - case OP_push: - print_indent(1); - printf("push %%%s", rs1->var_name); - break; - case OP_call: - print_indent(1); - printf("call @%s", insn->str); - break; - case OP_func_ret: - print_indent(1); - printf("retval %%%s", rd->var_name); - break; - case OP_return: - print_indent(1); - if (rs1) - printf("ret %%%s", rs1->var_name); - else - printf("ret"); - break; - case OP_read: - print_indent(1); - printf("%%%s = (%%%s), %d", rd->var_name, rs1->var_name, insn->sz); - break; - case OP_write: - print_indent(1); - if (rs1->is_func) - printf("(%%%s) = @%s", rs1->var_name, rs2->var_name); - else - printf("(%%%s) = %%%s, %d", rs1->var_name, rs2->var_name, - insn->sz); - break; - case OP_indirect: - print_indent(1); - printf("indirect call @(%%%s)", rs1->var_name); - break; - case OP_negate: - print_indent(1); - printf("neg %%%s, %%%s", rd->var_name, rs1->var_name); - break; - case OP_add: - print_indent(1); - printf("%%%s = add %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_sub: - print_indent(1); - printf("%%%s = sub %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_mul: - print_indent(1); - printf("%%%s = mul %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_div: - print_indent(1); - printf("%%%s = div %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_mod: - print_indent(1); - printf("%%%s = mod %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_eq: - print_indent(1); - printf("%%%s = eq %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_neq: - print_indent(1); - printf("%%%s = neq %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_gt: - print_indent(1); - printf("%%%s = gt %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_lt: - print_indent(1); - printf("%%%s = lt %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_geq: - print_indent(1); - printf("%%%s = geq %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_leq: - print_indent(1); - printf("%%%s = leq %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_bit_and: - print_indent(1); - printf("%%%s = and %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_bit_or: - print_indent(1); - printf("%%%s = or %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_bit_not: - print_indent(1); - printf("%%%s = not %%%s", rd->var_name, rs1->var_name); - break; - case OP_bit_xor: - print_indent(1); - printf("%%%s = xor %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_log_and: - print_indent(1); - printf("%%%s = and %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_log_or: - print_indent(1); - printf("%%%s = or %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_log_not: - print_indent(1); - printf("%%%s = not %%%s", rd->var_name, rs1->var_name); - break; - case OP_rshift: - print_indent(1); - printf("%%%s = rshift %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_lshift: - print_indent(1); - printf("%%%s = lshift %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_trunc: - print_indent(1); - printf("%%%s = trunc %%%s, %d", rd->var_name, rs1->var_name, - insn->sz); - break; - case OP_sign_ext: - print_indent(1); - printf("%%%s = sign_ext %%%s, %d", rd->var_name, rs1->var_name, - insn->sz); - break; - case OP_cast: - print_indent(1); - printf("%%%s = cast %%%s", rd->var_name, rs1->var_name); - break; - default: - printf("", insn->opcode); - break; - } - - printf("\n"); - } -} - -void dump_bb_insn_by_dom(func_t *func, basic_block_t *bb, bool *at_func_start) -{ - dump_bb_insn(func, bb, at_func_start); - for (int i = 0; bb && i < bb->dom_next_idx; i++) - dump_bb_insn_by_dom(func, bb->dom_next[i], at_func_start); -} - -void dump_insn(void) +int abi_arg_next(int cursor, const var_t *var, bool variadic) { - printf("====\n"); - - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - bool at_func_start = true; - - printf("def %s", func->return_def.type->type_name); - - for (int i = 0; i < func->return_def.ptr_level; i++) - printf("*"); - printf(" @%s(", func->return_def.var_name); - - for (int i = 0; i < func->num_params; i++) { - if (i != 0) - printf(", "); - printf("%s", func->param_defs[i].type->type_name); - - for (int k = 0; k < func->param_defs[i].ptr_level; k++) - printf("*"); - printf(" %%%s", func->param_defs[i].var_name); - } - printf(") {\n"); - - dump_bb_insn_by_dom(func, func->bbs, &at_func_start); - - /* Handle implicit return */ - for (int i = 0; func->exit && i < func->exit->prev_idx; i++) { - const basic_block_t *bb = func->exit->prev[i].bb; - if (!bb) - continue; - - if (func->return_def.type != TY_void) - continue; - - if (bb->insn_list.tail) - if (bb->insn_list.tail->opcode == OP_return) - continue; - - print_indent(1); - printf("ret\n"); - } - - printf("}\n"); - } - - printf("====\n"); + bool pair = PTR_SIZE == 4 && var && var->type && !var->ptr_level && + !var->array_size && !var->is_func && + var->type->base_type != TYPE_struct && + var->type->base_type != TYPE_union && var->type->size == 8; + if (pair && (ELF_MACHINE != ELF_MACHINE_RV32 || variadic || + cursor >= MAX_ARGS_IN_REG)) + cursor = ALIGN_UP(cursor, 2); + return cursor + (pair ? 2 : 1); } diff --git a/src/lexer.c b/src/lexer.c index 3a1fc434..fdef48d0 100644 --- a/src/lexer.c +++ b/src/lexer.c @@ -10,11 +10,6 @@ #include "defs.h" #include "globals.c" -/* Hash table constants */ -#define NUM_DIRECTIVES 12 -#define NUM_KEYWORDS 32 - -/* Token mapping structure for elegant initialization */ typedef struct { char *name; token_kind_t token; @@ -28,18 +23,25 @@ hashmap_t *KEYWORD_MAP = NULL; token_kind_t *directive_tokens_storage = NULL; token_kind_t *keyword_tokens_storage = NULL; +static void init_token_map(hashmap_t **map, + token_kind_t **storage, + const token_mapping_t *mappings, + int count, + int capacity) +{ + *map = hashmap_create(capacity); + *storage = arena_alloc(TOKEN_ARENA, count * sizeof(**storage)); + for (int i = 0; i < count; i++) { + (*storage)[i] = mappings[i].token; + hashmap_put_borrowed(*map, mappings[i].name, &(*storage)[i]); + } +} + void lex_init_directives(void) { if (DIRECTIVE_MAP) return; - DIRECTIVE_MAP = hashmap_create(16); /* Small capacity for directives */ - - /* Initialization using struct compound literals for elegance */ - directive_tokens_storage = - arena_alloc(GENERAL_ARENA, NUM_DIRECTIVES * sizeof(token_kind_t)); - - /* Use array compound literal for directive mappings */ token_mapping_t directives[] = { {"#define", T_cppd_define}, {"#elif", T_cppd_elif}, {"#else", T_cppd_else}, {"#endif", T_cppd_endif}, @@ -48,13 +50,8 @@ void lex_init_directives(void) {"#include", T_cppd_include}, {"#pragma", T_cppd_pragma}, {"#undef", T_cppd_undef}, {"#line", T_cppd_line}, }; - - /* hashmap insertion */ - for (int i = 0; i < NUM_DIRECTIVES; i++) { - directive_tokens_storage[i] = directives[i].token; - hashmap_put(DIRECTIVE_MAP, directives[i].name, - &directive_tokens_storage[i]); - } + init_token_map(&DIRECTIVE_MAP, &directive_tokens_storage, directives, + sizeof(directives) / sizeof(*directives), 16); } void lex_init_keywords(void) @@ -62,13 +59,6 @@ void lex_init_keywords(void) if (KEYWORD_MAP) return; - KEYWORD_MAP = hashmap_create(32); /* Capacity for keywords */ - - /* Initialization using struct compound literals for elegance */ - keyword_tokens_storage = - arena_alloc(GENERAL_ARENA, NUM_KEYWORDS * sizeof(token_kind_t)); - - /* Use array compound literal for keyword mappings */ token_mapping_t keywords[] = { {"if", T_if}, {"while", T_while}, @@ -103,12 +93,8 @@ void lex_init_keywords(void) {"_Complex", T_complex}, {"_Imaginary", T_imaginary}, }; - - /* hashmap insertion */ - for (int i = 0; i < NUM_KEYWORDS; i++) { - keyword_tokens_storage[i] = keywords[i].token; - hashmap_put(KEYWORD_MAP, keywords[i].name, &keyword_tokens_storage[i]); - } + init_token_map(&KEYWORD_MAP, &keyword_tokens_storage, keywords, + sizeof(keywords) / sizeof(*keywords), 32); } /* Hash table lookup for preprocessor directives */ @@ -125,8 +111,11 @@ token_kind_t lookup_directive(char *token) } /* Hash table lookup for C keywords */ -token_kind_t lookup_keyword(char *token) +token_kind_t lookup_keyword(char *token, int length) { + if (length < 2 || length > 10) + return T_identifier; + if (!KEYWORD_MAP) lex_init_keywords(); @@ -151,9 +140,8 @@ void lexer_cleanup(void) KEYWORD_MAP = NULL; } - /* Token storage arrays are allocated from GENERAL_ARENA and will be - * automatically freed when the arena is freed in global_release(). No need - * to explicitly free them here. + /* Token storage arrays share TOKEN_ARENA's parse lifetime and are released + * by release_token_arena(). */ directive_tokens_storage = NULL; keyword_tokens_storage = NULL; @@ -168,8 +156,8 @@ char trigraph_char_at(strbuf_t *buf, int pos) { char third; - if (buf->plain_source || pos + 2 >= buf->capacity || - buf->elements[pos] != '?' || buf->elements[pos + 1] != '?') + if (pos + 2 >= buf->capacity || buf->elements[pos] != '?' || + buf->elements[pos + 1] != '?') return '\0'; third = buf->elements[pos + 2]; switch (third) { @@ -218,8 +206,6 @@ int skip_splices(strbuf_t *buf, int pos) { int width; - if (buf->plain_source) - return pos; while (pos < buf->capacity) { width = source_char_width(buf, pos); if (source_char_at(buf, pos) != '\\' || @@ -230,18 +216,33 @@ int skip_splices(strbuf_t *buf, int pos) return pos; } +/* Where the next "??" or backslash-newline at or after @pos starts, or the end + * of @buf. Either may begin a trigraph or a splice; a "??" that turns out not + * to be one only costs the slow path a character early. + */ +static int next_phase_mark(strbuf_t *buf, int pos) +{ + for (; pos + 1 < buf->capacity; pos++) { + char c = buf->elements[pos]; + + if ((c == '?' && buf->elements[pos + 1] == '?') || + (c == '\\' && buf->elements[pos + 1] == '\n')) + return pos; + } + return buf->capacity; +} + +/* Translation phases 1 and 2 are the identity below @plain_end, so the + * per-character readers index the bytes directly there and only step through + * trigraphs and splices near one. The read cursor never moves backwards, so a + * mark found once stays ahead of it until the cursor passes it. + */ char peek_char(strbuf_t *buf, int offset) { int pos = buf->size; - if (buf->plain_source) { - if (pos < buf->capacity) { - pos += offset; - if (pos > buf->capacity) - pos = buf->capacity; - } - return pos >= buf->capacity ? '\0' : buf->elements[pos]; - } + if (pos + offset < buf->plain_end) + return buf->elements[pos + offset]; pos = skip_splices(buf, pos); @@ -256,11 +257,9 @@ char read_char(strbuf_t *buf) { int pos = buf->size; - if (buf->plain_source) { - if (pos + 1 >= buf->capacity) - return buf->elements[buf->capacity - 1]; + if (pos + 1 < buf->plain_end) { buf->size = pos + 1; - return buf->elements[buf->size]; + return buf->elements[pos + 1]; } pos = skip_splices(buf, pos); @@ -269,28 +268,11 @@ char read_char(strbuf_t *buf) return source_char_at(buf, buf->capacity - 1); pos += source_char_width(buf, pos); buf->size = skip_splices(buf, pos); + if (buf->size >= buf->plain_end) + buf->plain_end = next_phase_mark(buf, buf->size); return source_char_at(buf, buf->size); } -/* Translation phases 1 and 2 are the identity on a source with no "??" and no - * backslash-newline, which is nearly every file. Check that once per buffer so - * the per-character readers above can index its bytes directly instead of - * testing each position for a trigraph and a splice. - */ -void classify_plain_source(strbuf_t *buf) -{ - buf->plain_source = true; - for (int i = 0; i + 1 < buf->capacity; i++) { - char c = buf->elements[i]; - - if ((c == '?' && buf->elements[i + 1] == '?') || - (c == '\\' && buf->elements[i + 1] == '\n')) { - buf->plain_source = false; - return; - } - } -} - /* Fill @dst with @len bytes of @f, returning how many arrived. * * The whole file is wanted and its size is already known, so it is asked for in @@ -320,15 +302,14 @@ int file_read_all(FILE *f, char *dst, int len) } #endif -strbuf_t *read_file(const char *filename) +/* @filename's contents, or NULL when it cannot be opened. */ +strbuf_t *try_read_file(const char *filename) { FILE *f = fopen(filename, "rb"); strbuf_t *src; - if (!f) { - printf("filename: %s\n", filename); - fatal("source file cannot be found."); - } + if (!f) + return NULL; fseek(f, 0, SEEK_END); int len = ftell(f); @@ -342,17 +323,34 @@ strbuf_t *read_file(const char *filename) return src; } -strbuf_t *get_file_buf(char *filename) +/* @filename's contents, read once and kept for every later request; NULL when + * it cannot be opened, so that a caller with a better place to report the + * mistake can. + */ +strbuf_t *try_get_file_buf(char *filename) { strbuf_t *buf; - if (!hashmap_contains(SRC_FILE_MAP, filename)) { - buf = read_file(filename); - hashmap_put(SRC_FILE_MAP, filename, buf); - } else { - buf = hashmap_get(SRC_FILE_MAP, filename); - } + if (hashmap_contains(SRC_FILE_MAP, filename)) + return hashmap_get(SRC_FILE_MAP, filename); + buf = try_read_file(filename); + if (buf) + hashmap_put_borrowed(SRC_FILE_MAP, filename, buf); + return buf; +} +strbuf_t *get_file_buf(char *filename) +{ + strbuf_t *buf = try_get_file_buf(filename); + + /* A file that cannot be opened is a mistake in how the compiler was + * invoked, not a broken invariant, so it exits rather than abort()ing. + */ + if (!buf) { + printf("[Error]: cannot open source file '%s'\n", filename); + fflush(NULL); + exit(1); + } return buf; } @@ -371,6 +369,16 @@ token_t *new_token(token_kind_t kind, const source_location_t *loc, int len) return token; } +static token_t *new_token_at_column(source_location_t *loc, + token_kind_t kind, + int len) +{ + token_t *token = new_token(kind, loc, len); + + loc->column += len; + return token; +} + /* A '#' opens a directive only when nothing but white space precedes it on its * logical line. The physical column cannot tell: a backslash-newline puts a '#' * at column 1 of the next physical line while phase 2 has already joined it to @@ -440,9 +448,7 @@ token_t *lex_layout(strbuf_t *buf, source_location_t *loc, char ch) paste_len += source_char_width(buf, buf->size); read_char(buf); } - token = new_token(T_hashhash, loc, paste_len); - loc->column += paste_len; - return token; + return new_token_at_column(loc, T_hashhash, paste_len); } if (!lex_at_line_start) { @@ -450,9 +456,7 @@ token_t *lex_layout(strbuf_t *buf, source_location_t *loc, char ch) for (int i = 0; i < hash_chars; i++) read_char(buf); - token = new_token(T_hash, loc, hash_len); - loc->column += hash_len; - return token; + return new_token_at_column(loc, T_hash, hash_len); } int sz = 0, source_len = hash_len; @@ -511,9 +515,7 @@ token_t *lex_layout(strbuf_t *buf, source_location_t *loc, char ch) if (directive_kind == T_identifier) directive_kind = T_cppd_unknown; - token = new_token(directive_kind, loc, source_len); - loc->column += source_len; - return token; + return new_token_at_column(loc, directive_kind, source_len); } /* Leave a UCN to lex_word(); an ordinary backslash remains available for @@ -521,9 +523,7 @@ token_t *lex_layout(strbuf_t *buf, source_location_t *loc, char ch) */ if (ch == '\\' && peek_char(buf, 1) != 'u' && peek_char(buf, 1) != 'U') { read_char(buf); - token = new_token(T_backslash, loc, 1); - loc->column++; - return token; + return new_token_at_column(loc, T_backslash, 1); } if (ch == '\n') { @@ -577,14 +577,10 @@ token_t *lex_layout(strbuf_t *buf, source_location_t *loc, char ch) if (ch == '=') { ch = read_char(buf); - token = new_token(T_divideeq, loc, 2); - loc->column += 2; - return token; + return new_token_at_column(loc, T_divideeq, 2); } - token = new_token(T_divide, loc, 1); - loc->column++; - return token; + return new_token_at_column(loc, T_divide, 1); } if (ch == ' ') { @@ -594,23 +590,17 @@ token_t *lex_layout(strbuf_t *buf, source_location_t *loc, char ch) while (read_char(buf) == ' ') sz++; - token = new_token(T_whitespace, loc, sz); - loc->column += sz; - return token; + return new_token_at_column(loc, T_whitespace, sz); } if (ch == '\t') { read_char(buf); - token = new_token(T_tab, loc, 1); - loc->column++; - return token; + return new_token_at_column(loc, T_tab, 1); } if (ch == '\0') { read_char(buf); - token = new_token(T_eof, loc, 1); - loc->column++; - return token; + return new_token_at_column(loc, T_eof, 1); } return NULL; @@ -930,118 +920,70 @@ token_t *lex_literal(strbuf_t *buf, source_location_t *loc, char ch) */ token_t *lex_punct(strbuf_t *buf, source_location_t *loc, char ch) { - token_t *token; - - if (ch == '(') { - ch = read_char(buf); - token = new_token(T_open_bracket, loc, 1); - loc->column++; - return token; - } - - if (ch == ')') { - ch = read_char(buf); - token = new_token(T_close_bracket, loc, 1); - loc->column++; - return token; - } - - if (ch == '{') { - ch = read_char(buf); - token = new_token(T_open_curly, loc, 1); - loc->column++; - return token; - } - if (ch == '<' && peek_char(buf, 1) == '%') { read_char(buf); read_char(buf); - token = new_token(T_open_curly, loc, 2); - loc->column += 2; - return token; - } - - if (ch == '}') { - ch = read_char(buf); - token = new_token(T_close_curly, loc, 1); - loc->column++; - return token; + return new_token_at_column(loc, T_open_curly, 2); } if (ch == '%' && peek_char(buf, 1) == '>') { read_char(buf); read_char(buf); - token = new_token(T_close_curly, loc, 2); - loc->column += 2; - return token; - } - - if (ch == '[') { - ch = read_char(buf); - token = new_token(T_open_square, loc, 1); - loc->column++; - return token; + return new_token_at_column(loc, T_close_curly, 2); } if (ch == '<' && peek_char(buf, 1) == ':') { read_char(buf); read_char(buf); - token = new_token(T_open_square, loc, 2); - loc->column += 2; - return token; - } - - if (ch == ']') { - ch = read_char(buf); - token = new_token(T_close_square, loc, 1); - loc->column++; - return token; + return new_token_at_column(loc, T_open_square, 2); } if (ch == ':' && peek_char(buf, 1) == '>') { read_char(buf); read_char(buf); - token = new_token(T_close_square, loc, 2); - loc->column += 2; - return token; - } - - if (ch == ',') { - ch = read_char(buf); - token = new_token(T_comma, loc, 1); - loc->column++; - return token; - } - - if (ch == '~') { - ch = read_char(buf); - token = new_token(T_bit_not, loc, 1); - loc->column++; - return token; - } - - if (ch == ';') { - read_char(buf); - token = new_token(T_semicolon, loc, 1); - loc->column++; - return token; - } - - if (ch == '?') { - read_char(buf); - token = new_token(T_question, loc, 1); - loc->column++; - return token; + return new_token_at_column(loc, T_close_square, 2); } - if (ch == ':') { - read_char(buf); - token = new_token(T_colon, loc, 1); - loc->column++; - return token; + token_kind_t kind; + switch (ch) { + case '(': + kind = T_open_bracket; + break; + case ')': + kind = T_close_bracket; + break; + case '{': + kind = T_open_curly; + break; + case '}': + kind = T_close_curly; + break; + case '[': + kind = T_open_square; + break; + case ']': + kind = T_close_square; + break; + case ',': + kind = T_comma; + break; + case '~': + kind = T_bit_not; + break; + case ';': + kind = T_semicolon; + break; + case '?': + kind = T_question; + break; + case ':': + kind = T_colon; + break; + default: + return NULL; } - - return NULL; + read_char(buf); + return new_token_at_column(loc, kind, 1); } /* Operators, each of which may or may not continue into a longer one. @@ -1049,254 +991,114 @@ token_t *lex_punct(strbuf_t *buf, source_location_t *loc, char ch) * Returns NULL when 'ch' is none of its business, so that lex_token() can offer * the character to the next reader in line. */ -token_t *lex_operator(strbuf_t *buf, source_location_t *loc, char ch) +static token_t *lex_short_operator(strbuf_t *buf, + source_location_t *loc, + char pair_char, + token_kind_t pair_kind, + char alternate_char, + token_kind_t alternate_kind, + token_kind_t assign_kind, + token_kind_t single_kind) { - token_t *token; - - if (ch == '^') { - ch = read_char(buf); - - if (ch == '=') { - ch = read_char(buf); - token = new_token(T_xoreq, loc, 2); - loc->column += 2; - return token; - } - - token = new_token(T_bit_xor, loc, 1); - loc->column++; - return token; - } - - if (ch == '*') { - ch = read_char(buf); + int length = 1; + token_kind_t kind = single_kind; - if (ch == '=') { - read_char(buf); - token = new_token(T_asteriskeq, loc, 2); - loc->column += 2; - return token; - } - - token = new_token(T_asterisk, loc, 1); - loc->column++; - return token; - } - - if (ch == '&') { - ch = read_char(buf); - - if (ch == '&') { - read_char(buf); - token = new_token(T_log_and, loc, 2); - loc->column += 2; - return token; - } - - if (ch == '=') { - read_char(buf); - token = new_token(T_andeq, loc, 2); - loc->column += 2; - return token; - } - - token = new_token(T_ampersand, loc, 1); - loc->column++; - return token; - } - - if (ch == '|') { - ch = read_char(buf); - - if (ch == '|') { - read_char(buf); - token = new_token(T_log_or, loc, 2); - loc->column += 2; - return token; - } - - if (ch == '=') { - read_char(buf); - token = new_token(T_oreq, loc, 2); - loc->column += 2; - return token; - } - - token = new_token(T_bit_or, loc, 1); - loc->column++; - return token; - } - - if (ch == '<') { - ch = read_char(buf); - - if (ch == '=') { - read_char(buf); - token = new_token(T_le, loc, 2); - loc->column += 2; - return token; - } - - if (ch == '<') { - ch = read_char(buf); - - if (ch == '=') { - read_char(buf); - token = new_token(T_lshifteq, loc, 3); - loc->column += 3; - return token; - } - - token = new_token(T_lshift, loc, 2); - loc->column += 2; - return token; - } - - token = new_token(T_lt, loc, 1); - loc->column++; - return token; - } - - if (ch == '%') { - ch = read_char(buf); - - if (ch == '=') { - read_char(buf); - token = new_token(T_modeq, loc, 2); - loc->column += 2; - return token; - } + char ch = read_char(buf); + if (pair_char && ch == pair_char) { + kind = pair_kind; + length++; + } else if (alternate_char && ch == alternate_char) { + kind = alternate_kind; + length++; + } else if (ch == '=' && assign_kind != T_start) { + kind = assign_kind; + length++; + } + if (length == 2) + read_char(buf); + return new_token_at_column(loc, kind, length); +} - token = new_token(T_mod, loc, 1); - loc->column++; - return token; - } +static token_t *lex_angle_operator(strbuf_t *buf, + source_location_t *loc, + char first, + token_kind_t single, + token_kind_t equal, + token_kind_t shift, + token_kind_t shift_equal) +{ + char ch = read_char(buf); + int length = 1; + token_kind_t kind = single; - if (ch == '>') { + if (ch == '=') { + kind = equal; + length = 2; + read_char(buf); + } else if (ch == first) { ch = read_char(buf); - + kind = shift; + length = 2; if (ch == '=') { + kind = shift_equal; + length++; read_char(buf); - token = new_token(T_ge, loc, 2); - loc->column += 2; - return token; } - - if (ch == '>') { - ch = read_char(buf); - - if (ch == '=') { - read_char(buf); - token = new_token(T_rshifteq, loc, 3); - loc->column += 3; - return token; - } - - token = new_token(T_rshift, loc, 2); - loc->column += 2; - return token; - } - - token = new_token(T_gt, loc, 1); - loc->column++; - return token; } + return new_token_at_column(loc, kind, length); +} - if (ch == '!') { - ch = read_char(buf); - - if (ch == '=') { - read_char(buf); - token = new_token(T_noteq, loc, 2); - loc->column += 2; - return token; - } - - token = new_token(T_log_not, loc, 1); - loc->column++; - return token; - } +token_t *lex_operator(strbuf_t *buf, source_location_t *loc, char ch) +{ + if (ch == '^') + return lex_short_operator(buf, loc, 0, T_start, 0, T_start, T_xoreq, + T_bit_xor); + if (ch == '*') + return lex_short_operator(buf, loc, 0, T_start, 0, T_start, + T_asteriskeq, T_asterisk); + if (ch == '&') + return lex_short_operator(buf, loc, '&', T_log_and, 0, T_start, T_andeq, + T_ampersand); + if (ch == '|') + return lex_short_operator(buf, loc, '|', T_log_or, 0, T_start, T_oreq, + T_bit_or); + + if (ch == '<') + return lex_angle_operator(buf, loc, '<', T_lt, T_le, T_lshift, + T_lshifteq); + + if (ch == '%') + return lex_short_operator(buf, loc, 0, T_start, 0, T_start, T_modeq, + T_mod); + + if (ch == '>') + return lex_angle_operator(buf, loc, '>', T_gt, T_ge, T_rshift, + T_rshifteq); + + if (ch == '!') + return lex_short_operator(buf, loc, 0, T_start, 0, T_start, T_noteq, + T_log_not); if (ch == '.') { ch = read_char(buf); if (ch == '.' && peek_char(buf, 1) == '.') { buf->size += 2; - token = new_token(T_elipsis, loc, 3); - loc->column += 3; - return token; - } - - token = new_token(T_dot, loc, 1); - loc->column++; - return token; - } - - if (ch == '-') { - ch = read_char(buf); - - if (ch == '>') { - read_char(buf); - token = new_token(T_arrow, loc, 2); - loc->column += 2; - return token; - } - - if (ch == '-') { - read_char(buf); - token = new_token(T_decrement, loc, 2); - loc->column += 2; - return token; + return new_token_at_column(loc, T_elipsis, 3); } - if (ch == '=') { - read_char(buf); - token = new_token(T_minuseq, loc, 2); - loc->column += 2; - return token; - } - - token = new_token(T_minus, loc, 1); - loc->column++; - return token; - } - - if (ch == '+') { - ch = read_char(buf); - - if (ch == '+') { - read_char(buf); - token = new_token(T_increment, loc, 2); - loc->column += 2; - return token; - } - - if (ch == '=') { - read_char(buf); - token = new_token(T_pluseq, loc, 2); - loc->column += 2; - return token; - } - - token = new_token(T_plus, loc, 1); - loc->column++; - return token; + return new_token_at_column(loc, T_dot, 1); } - if (ch == '=') { - ch = read_char(buf); - - if (ch == '=') { - read_char(buf); - token = new_token(T_eq, loc, 2); - loc->column += 2; - return token; - } - - token = new_token(T_assign, loc, 1); - loc->column++; - return token; - } + if (ch == '-') + return lex_short_operator(buf, loc, '-', T_decrement, '>', T_arrow, + T_minuseq, T_minus); + if (ch == '+') + return lex_short_operator(buf, loc, '+', T_increment, 0, T_start, + T_pluseq, T_plus); + if (ch == '=') + return lex_short_operator(buf, loc, '=', T_eq, 0, T_start, T_start, + T_assign); return NULL; } @@ -1318,7 +1120,6 @@ static int lex_ucn_identifier(char *out, int out_size, strbuf_t *buf) { int digits = peek_char(buf, 1) == 'u' ? 4 : 8; unsigned int value = 0; - int out_len; for (int i = 0; i < digits; i++) { int digit = hex_digit_value(peek_char(buf, i + 2)); @@ -1337,33 +1138,7 @@ static int lex_ucn_identifier(char *out, int out_size, strbuf_t *buf) value > 0x10ffff || (value >= 0xd800 && value <= 0xdfff)) return -1; - if (value <= 0x7f) - out_len = 1; - else if (value <= 0x7ff) - out_len = 2; - else if (value <= 0xffff) - out_len = 3; - else - out_len = 4; - if (out_len >= out_size) - return -1; - - if (out_len == 1) { - out[0] = value; - } else if (out_len == 2) { - out[0] = 0xc0 | (value >> 6); - out[1] = 0x80 | (value & 0x3f); - } else if (out_len == 3) { - out[0] = 0xe0 | (value >> 12); - out[1] = 0x80 | ((value >> 6) & 0x3f); - out[2] = 0x80 | (value & 0x3f); - } else { - out[0] = 0xf0 | (value >> 18); - out[1] = 0x80 | ((value >> 12) & 0x3f); - out[2] = 0x80 | ((value >> 6) & 0x3f); - out[3] = 0x80 | (value & 0x3f); - } - return out_len; + return append_utf8(out, 0, out_size, value); } /* Identifiers, and the keywords spelled like them. @@ -1413,113 +1188,7 @@ token_t *lex_word(strbuf_t *buf, source_location_t *loc, char ch) } while (isalnum(ch) || ch == '_' || lex_ucn_starts(buf)); token_buffer[sz] = 0; - /* Fast path for common keywords - avoid hashmap lookup */ - token_kind_t kind = T_identifier; - - /* Check most common keywords inline based on token length and first - * character. - */ - switch (sz) { - case 2: /* 2-letter keywords: if, do */ - if (token_buffer[0] == 'i' && token_buffer[1] == 'f') - kind = T_if; - else if (token_buffer[0] == 'd' && token_buffer[1] == 'o') - kind = T_do; - break; - - case 3: /* 3-letter keywords: for */ - if (token_buffer[0] == 'f' && token_buffer[1] == 'o' && - token_buffer[2] == 'r') - kind = T_for; - break; - - case 4: /* 4-letter keywords: else, enum, case */ - if (token_buffer[0] == 'e') { - if (!memcmp(token_buffer, "else", 4)) - kind = T_else; - else if (!memcmp(token_buffer, "enum", 4)) - kind = T_enum; - } else if (!memcmp(token_buffer, "case", 4)) - kind = T_case; - else if (!memcmp(token_buffer, "goto", 4)) - kind = T_goto; - break; - - case 5: /* 5-letter keywords: while, break, union, const, float */ - if (token_buffer[0] == 'w' && !memcmp(token_buffer, "while", 5)) - kind = T_while; - else if (token_buffer[0] == 'b' && - !memcmp(token_buffer, "break", 5)) - kind = T_break; - else if (token_buffer[0] == 'u' && - !memcmp(token_buffer, "union", 5)) - kind = T_union; - else if (token_buffer[0] == 'c' && - !memcmp(token_buffer, "const", 5)) - kind = T_const; - else if (token_buffer[0] == 'f' && - !memcmp(token_buffer, "float", 5)) - kind = T_float; - break; - - case 6: /* 6-letter keywords: return, struct, switch, sizeof, static, - extern, inline - */ - if (token_buffer[0] == 'r' && !memcmp(token_buffer, "return", 6)) - kind = T_return; - else if (token_buffer[0] == 'e' && - !memcmp(token_buffer, "extern", 6)) - kind = T_extern; - else if (token_buffer[0] == 'i' && - !memcmp(token_buffer, "inline", 6)) - kind = T_inline; - else if (token_buffer[0] == 'd' && - !memcmp(token_buffer, "double", 6)) - kind = T_double; - else if (token_buffer[0] == 's') { - if (!memcmp(token_buffer, "struct", 6)) - kind = T_struct; - else if (!memcmp(token_buffer, "switch", 6)) - kind = T_switch; - else if (!memcmp(token_buffer, "sizeof", 6)) - kind = T_sizeof; - else if (!memcmp(token_buffer, "static", 6)) - kind = T_static; - } - break; - - case 7: /* 7-letter keywords: typedef, default */ - if (!memcmp(token_buffer, "typedef", 7)) - kind = T_typedef; - else if (!memcmp(token_buffer, "default", 7)) - kind = T_default; - break; - - case 8: /* 8-letter keywords: continue, register, restrict, volatile */ - if (!memcmp(token_buffer, "continue", 8)) - kind = T_continue; - else if (!memcmp(token_buffer, "register", 8)) - kind = T_register; - else if (!memcmp(token_buffer, "restrict", 8)) - kind = T_restrict; - else if (!memcmp(token_buffer, "volatile", 8)) - kind = T_volatile; - break; - - default: - /* Keywords longer than 8 chars or identifiers - use hashmap */ - break; - } - - /* Fall back to the hashmap for anything the switch does not name. No - * keyword is shorter than two characters or longer than eight, so a - * name outside that range cannot be one and needs no lookup -- which is - * most of the identifiers in a real program. - */ - if (kind == T_identifier && !strcmp(token_buffer, "_Imaginary")) - kind = T_imaginary; - else if (kind == T_identifier && sz >= 2 && sz <= 8) - kind = lookup_keyword(token_buffer); + token_kind_t kind = lookup_keyword(token_buffer, sz); token = new_token(kind, loc, source_len); token->literal = intern_string(token_buffer); @@ -1540,7 +1209,8 @@ token_t *lex_token(strbuf_t *buf, source_location_t *loc) token_t *token; char ch; - while (source_char_at(buf, buf->size) == '\\' && + while (buf->size >= buf->plain_end && + source_char_at(buf, buf->size) == '\\' && source_char_at(buf, buf->size + source_char_width(buf, buf->size)) == '\n') { buf->size += source_char_width(buf, buf->size) + 1; @@ -1688,6 +1358,18 @@ static char *normalize_filename(const char *filename) return intern_string(path); } +static token_stream_t *cache_token_stream(char *filename, + token_t *head, + token_t *tail) +{ + token_stream_t *stream = malloc(sizeof(*stream)); + + stream->head = head; + stream->tail = tail; + hashmap_put_borrowed(TOKEN_CACHE, filename, stream); + return stream; +} + token_stream_t *gen_file_token_stream(char *filename) { token_t head; @@ -1709,7 +1391,7 @@ token_stream_t *gen_file_token_stream(char *filename) return tks; buf = get_file_buf(filename); - classify_plain_source(buf); + buf->plain_end = next_phase_mark(buf, 0); /* Borrows strbuf_t#size to use as source index */ buf->size = 0; @@ -1734,11 +1416,7 @@ token_stream_t *gen_file_token_stream(char *filename) error_at("Internal error, expected eof at the end of file", &cur->location); - tks = malloc(sizeof(token_stream_t)); - tks->head = head.next; - tks->tail = cur; - hashmap_put(TOKEN_CACHE, filename, tks); - return tks; + return cache_token_stream(filename, head.next, cur); } token_stream_t *gen_libc_token_stream(void) @@ -1756,12 +1434,12 @@ token_stream_t *gen_libc_token_stream(void) return tks; if (!hashmap_contains(SRC_FILE_MAP, filename)) - hashmap_put(SRC_FILE_MAP, filename, LIBC_SRC); + hashmap_put_borrowed(SRC_FILE_MAP, filename, LIBC_SRC); /* This buffer was built by appending, so its capacity is whatever the * doubling left and runs past the text into memory that was never written * -- while the scan below, like the one over a file, stops at capacity. - * Terminate it the way read_file() leaves a file: the text, a NUL, and + * Terminate it the way try_read_file() leaves a file: the text, a NUL, and * capacity naming one past the text. Without this the lexer reads * uninitialised bytes, and what it finds there depends on the allocator, * which is enough to make the compiler emit different code from one build @@ -1770,7 +1448,7 @@ token_stream_t *gen_libc_token_stream(void) if (!buf->size || buf->elements[buf->size - 1]) strbuf_putc(buf, 0); buf->capacity = buf->size; - classify_plain_source(buf); + buf->plain_end = next_phase_mark(buf, 0); /* Borrows strbuf_t#size to use as source index */ buf->size = 0; @@ -1796,11 +1474,7 @@ token_stream_t *gen_libc_token_stream(void) error_at("Internal error, expected eof at the end of file", &cur->location); - tks = malloc(sizeof(token_stream_t)); - tks->head = head.next; - tks->tail = cur; - hashmap_put(TOKEN_CACHE, filename, tks); - return tks; + return cache_token_stream(filename, head.next, cur); } /* Fetches current token's location. */ diff --git a/src/main.c b/src/main.c index 24fc2fe0..87de3d54 100644 --- a/src/main.c +++ b/src/main.c @@ -9,12 +9,17 @@ #include #include #include +#ifdef __GLIBC__ +#include +#endif /* Define target machine */ #include "../config" /* The inclusion must follow the fixed order, otherwise it fails to build. */ #include "defs.h" +#include "vir-frontend.h" +#include "vir-lower.h" /* Initialize global objects */ #include "globals.c" @@ -31,18 +36,16 @@ /* C language syntactic analyzer */ #include "parser.c" -/* architecture-independent middle-end */ -#include "ssa.c" +/* Native value IR construction and optimization. */ +#include "vir-frontend.c" +#include "vir.c" /* Register allocator */ -#include "reg-alloc.c" +#include "vir-lower.c" -/* Peephole optimization */ +/* Machine peepholes, liveness and target lowering */ #include "peephole.c" -/* Arch-specific IR lowering boundary */ -#include "arch-lower.c" - /* Machine code generation. support ARMv7-A and RV32I */ #include "codegen.c" @@ -99,30 +102,226 @@ char *dot_output_name(char *input) return output; } + +/* Functions marked used but not yet scanned. Each enters once, when it is first + * marked, so reachability costs one visit per function. + */ +static func_t **native_worklist; +static int native_worklist_count; + +static void mark_native_used(func_t *func) +{ + if (!func || func->is_used) + return; + func->is_used = true; + native_worklist[native_worklist_count++] = func; +} + +static void mark_native_references(const vir_function_t *graph) +{ + for (const vir_block_t *block = graph->blocks; block; block = block->next) { + for (const vir_effect_t *effect = block->effects; effect; + effect = effect->next) + if (effect->kind == VIR_EFFECT_CALL && effect->callee) + mark_native_used(find_func((char *) effect->callee)); + for (const vir_value_t *value = block->head; value; value = value->next) + if (value->opcode == VIR_OP_FUNC_ADDR) + mark_native_used(find_func((char *) value->addr->name)); + } +} + +static void lower_native_function(func_t *func, bool global) +{ + vir_function_t *graph = (vir_function_t *) vir_frontend_function(func); + vir_machine_function_t machine = {0}; + int global_bytes = GLOBAL_FUNC->stack_size; + if (!vir_lower_machine(graph, vir_frontend_signature(func), + vir_frontend_global_offset, NULL, &machine)) + fatal("VIR machine lowering failed"); + func->bbs = machine.entry; + if (!global) { + func->stack_size = machine.stack_size; + func->passes_stack_args = machine.passes_stack_args; + if (!strcmp(func->return_def.var_name, "main")) + MAIN_BB = machine.entry; + } else { + int scratch_base = + machine.stack_size ? (global_bytes + 7) & -8 : global_bytes; + if (machine.block_count != 1) + fatal("Unexpected control flow in global initializer"); + vir_mop_t *previous = NULL; + for (vir_mop_t *ir = machine.entry->mops.head; ir; ir = ir->next) { + if (ir->op == OP_load) { + ir->op = OP_global_load; + ir->src0 += scratch_base; + } else if (ir->op == OP_store) { + ir->op = OP_global_store; + ir->src1 += scratch_base; + } else if (ir->op == OP_address_of) { + ir->op = OP_global_address_of; + ir->src0 += scratch_base; + } else if (ir->op == OP_return) { + if (previous) + previous->next = NULL; + else + machine.entry->mops.head = NULL; + machine.entry->mops.tail = previous; + break; + } + previous = ir; + } + func->stack_size = scratch_base + machine.stack_size; + } + for (basic_block_t *block = machine.entry; block; block = block->rpo_next) + block->belong_to = func; + + /* The machine blocks own copies of everything emission reads. */ + vir_frontend_release_function(func); +} + +static void compile_native_vir(const char *output) +{ + int function_count = 0; + for (func_t *func = FUNC_LIST.head; func; func = func->next) { + func->is_used = false; + function_count++; + } + native_worklist = malloc((function_count + 1) * sizeof(func_t *)); + if (!native_worklist) + fatal("Out of memory"); + vir_frontend_layout_globals(); + vir_function_t *global = + (vir_function_t *) vir_frontend_function(GLOBAL_FUNC); + if (global) { + vir_frontend_optimize(global); + mark_native_references(global); + } + func_t *main_func = find_func("main"); + if (main_func) + mark_native_used(main_func); + else + for (func_t *func = FUNC_LIST.head; func; func = func->next) + if (vir_frontend_function(func)) + mark_native_used(func); + while (native_worklist_count) { + func_t *func = native_worklist[--native_worklist_count]; + vir_function_t *graph = (vir_function_t *) vir_frontend_function(func); + if (graph) + mark_native_references(graph); + } + free(native_worklist); + /* Report the optimized graphs here: lowering releases each one. */ + dump_stats_phase("vir"); + FILE *dot = dump_dot ? fopen(output, "w") : NULL; + if (dump_dot && !dot) + fatal("Cannot open DOT output"); + if (dot) + fprintf(dot, "digraph VIR {\n"); + int function_index = 0; + for (func_t *func = FUNC_LIST.head; func; + func = func->next, function_index++) { + vir_function_t *graph = (vir_function_t *) vir_frontend_function(func); + if (func->is_used && !graph) { + char message[MAX_LINE_LEN]; + if (func->is_static || !dynlink) { + snprintf(message, MAX_LINE_LEN, "undefined %sfunction '%s'", + func->is_static ? "static " : "", + function_source_name(func)); + error_at(message, NULL); + } + if (func->returns_aggregate) { + snprintf(message, MAX_LINE_LEN, + "aggregate-return function '%s' has its address taken " + "but is not defined", + func->return_def.var_name); + error_at(message, NULL); + } + } + if (!func->is_used || !graph) { + vir_frontend_release_function(func); + func->bbs = NULL; + continue; + } + if (dump_ir || dump_vir) { + FILE *stream = dump_ir ? stdout : stderr; + fprintf(stream, "function %s\n", func->return_def.var_name); + vir_print(graph, stream); + } + if (dot) { + for (vir_block_t *block = graph->blocks; block; + block = block->next) { + fprintf(dot, "f%d_b%d [label=\"%s block %d\"];\n", + function_index, block->id, func->return_def.var_name, + block->id); + for (vir_edge_t *edge = block->outgoing; edge; + edge = edge->next_outgoing) + fprintf(dot, "f%d_b%d -> f%d_b%d;\n", function_index, + block->id, function_index, edge->to->id); + } + } else { + lower_native_function(func, false); + } + } + if (dot) { + fprintf(dot, "}\n"); + fclose(dot); + } else if (global) { + lower_native_function(GLOBAL_FUNC, true); + } +} + int main(int argc, char *argv[]) { + static const struct { + const char *name; + bool *setting; + bool value; + } flags[] = { + {"--dump-ir", &dump_ir, true}, + {"--stats", &dump_stats, true}, + {"--dump-vir", &dump_vir, true}, + {"--warn-string-literals", &warn_string_literals, true}, + {"--std=c99", &strict_c99, true}, + {"--dot", &dump_dot, true}, + {"+m", &hard_mul_div, true}, + {"--no-libc", &libc, false}, + {"--no-opt", &optimize, false}, + {"--dynlink", &dynlink, true}, + {"-E", &expand_only, true}, + }; char *out = NULL; - char *in = NULL; + char **inputs; + int input_count = 0; token_stream_t *libc_token_stream = NULL, *token_stream; - token_t *tk; + token_t *input_tokens = NULL, *input_tail = NULL; bool has_binding_option = false; +#ifdef __GLIBC__ + /* glibc maps each large block on its own and hands back the top of the heap + * as soon as it is free, so the tens of megabytes of tokens dropped before + * parsing go back to the system, and the parser faults in fresh pages for + * what it builds in their place. Keeping freed memory for reuse removes two + * fifths of the page faults without raising the peak. + */ + mallopt(M_MMAP_THRESHOLD, 16 * 1024 * 1024); + mallopt(M_TRIM_THRESHOLD, 64 * 1024 * 1024); +#endif + + inputs = malloc((size_t) argc * sizeof(*inputs)); + if (!inputs) + fatal("Unable to allocate input file list"); + for (int i = 1; i < argc; i++) { - if (!strcmp(argv[i], "--dump-ir")) - dump_ir = true; - else if (!strcmp(argv[i], "--warn-string-literals")) - warn_string_literals = true; - else if (!strcmp(argv[i], "--std=c99")) - strict_c99 = true; - else if (!strcmp(argv[i], "--dot")) - dump_dot = true; - else if (!strcmp(argv[i], "+m")) - hard_mul_div = true; - else if (!strcmp(argv[i], "--no-libc")) - libc = false; - else if (!strcmp(argv[i], "--dynlink")) - dynlink = true; - else if (!strcmp(argv[i], "-z")) { + size_t flag = 0; + + while (flag < sizeof(flags) / sizeof(*flags) && + strcmp(argv[i], flags[flag].name)) + flag++; + if (flag < sizeof(flags) / sizeof(*flags)) { + *flags[flag].setting = flags[flag].value; + continue; + } + if (!strcmp(argv[i], "-z")) { if (i + 1 >= argc) usage_error("-z requires \"lazy\" or \"now\""); @@ -134,9 +333,7 @@ int main(int argc, char *argv[]) else usage_error("-z requires \"lazy\" or \"now\""); i++; - } else if (!strcmp(argv[i], "-E")) - expand_only = true; - else if (!strcmp(argv[i], "-I")) { + } else if (!strcmp(argv[i], "-I")) { if (i + 1 >= argc) usage_error("-I requires an include directory"); if (include_dirs_idx == MAX_INCLUDE_DIRS) @@ -155,15 +352,16 @@ int main(int argc, char *argv[]) } else if (argv[i][0] == '-') { usage_error("Unidentified option"); } else - in = argv[i]; + inputs[input_count++] = argv[i]; } - if (!in) { + if (!input_count) { printf( - "Usage: shecc [-I directory] [-o output] [+m] [--dot] " - "[--dump-ir] " + "Usage: shecc [-I directory] [-o output] [+m] [--dot] [--dump-ir] " + "[--dump-vir] [--stats] " + "[--no-opt] " "[--warn-string-literals] [--std=c99] [--no-libc] " - "[--dynlink] [-z ] [-E] \n"); + "[--dynlink] [-z ] [-E] [input.c ...]\n"); usage_error("Missing source file"); } @@ -177,18 +375,25 @@ int main(int argc, char *argv[]) if (dump_dot && dump_ir) usage_error("--dot cannot be combined with --dump-ir"); - if (dump_dot && !out) - out = dot_output_name(in); + if (dump_dot && !out) { + if (input_count != 1) + usage_error("--dot requires -o with multiple input files"); + out = dot_output_name(inputs[0]); + } /* The graph is written by truncating its output, so naming the input * destroys the source. That happens both when -o names it outright and when * an input already ending in .dot derives its own name. */ - if (dump_dot && !strcmp(out, in)) - usage_error("--dot would overwrite the input; name another output"); + if (dump_dot) + for (int i = 0; i < input_count; i++) + if (!strcmp(out, inputs[i])) + usage_error( + "--dot would overwrite an input; name another output"); /* initialize global objects */ global_init(); + dump_stats_layout(); /* include libc */ if (libc) { @@ -198,118 +403,86 @@ int main(int argc, char *argv[]) libc_token_stream = gen_libc_token_stream(); } - token_stream = gen_file_token_stream(in); - - /* concat libc's and input file's token stream */ - if (libc_token_stream) { - libc_token_stream->tail->next = token_stream->head; - token_stream = libc_token_stream; + /* Preprocess each input independently so macro definitions and #pragma once + * state do not leak between translation units. The existing implicit libc + * prelude remains attached to the first input only. + */ + for (int i = 0; i < input_count; i++) { + token_t *unit_tokens; + + token_stream = gen_file_token_stream(inputs[i]); + if (i == 0 && libc_token_stream) { + libc_token_stream->tail->next = token_stream->head; + token_stream = libc_token_stream; + } + unit_tokens = preprocess(token_stream->head); + if (!unit_tokens) + fatal("Preprocessor produced no input tokens"); + if (input_tail) { + /* An EOF ends preprocessing for one input, but not compilation of + * the next input. Reuse it as an explicit parser-visible boundary + * that also preserves the line separator emitted by -E. + */ + input_tail->kind = T_translation_unit; + input_tail->next = unit_tokens; + } else { + input_tokens = unit_tokens; + } + input_tail = unit_tokens; + while (input_tail->kind != T_eof && input_tail->next) + input_tail = input_tail->next; + if (input_tail->kind != T_eof) + fatal("Preprocessor output has no end-of-file token"); } - tk = preprocess(token_stream->head); - if (expand_only) { - emit_preprocessed_token(tk); + emit_preprocessed_token(input_tokens); + free(inputs); exit(0); } - /* load and parse source code into IR */ - parse(pp_strip_layout(tk)); + /* load and parse source code into IR; each input is one unit */ + translation_unit_count = input_count; + parse(compact_token_stream(pp_strip_layout(input_tokens))); + free(inputs); + dump_stats_phase("parse"); - /* Tokens, macros and the source buffers they point into are dead once - * parsing finishes; releasing them here keeps 17 MiB out of the peak. + /* The token stream and the source buffers it points into are dead once + * parsing finishes. */ release_token_arena(); - /* Compact arenas after parsing to free temporary parse structures */ - compact_all_arenas(); - - ssa_build(); - - /* Like lacc's -dot target, visualize the SSA CFG directly rather than - * lowering it into an executable. Prune first: every compile prepends the - * whole of lib/c.c, and the functions nothing reaches are noise in the - * picture -- two thirds of the graph on tests/fib.c. - */ + /* Finalize graphs after all declarations and initializers are known. */ + vir_frontend_finish_globals(); + release_parse_blocks(); + compile_native_vir(out); if (dump_dot) { - prune_unused_funcs(); - dump_cfg(out); + vir_frontend_release(); global_release(); exit(0); } - unwind_phi(); - - /* Copy small helpers into their callers before anything else looks at them, - * so the optimizer sees one body rather than a call boundary. - */ - inline_calls(); - - /* dump first phase IR */ - if (dump_ir) - dump_insn(); - - /* Nothing beyond this point should spend time on functions no call can - * reach; on a small input, almost all of the embedded libc is that. - */ - prune_unused_funcs(); - - /* SSA-based optimization */ - optimize(); - - /* Flatten unpredictable ifs into branchless selects. - * - * After the optimizer rather than inside ssa_build(): a select reads a - * third operand, and the passes in optimize() walk instructions two sources - * at a time. One of them would rewrite a copy feeding that third operand - * and leave the select reading a value nothing defines. - */ - if_convert(); - - /* Send short-circuit arms straight to their destination. */ - thread_const_branches(); - - /* Both passes above rewired the CFG the dominator tree was built from, and - * everything below asks that tree which edges close a loop. - */ - rebuild_dom(); - - /* Walk arrays with a pointer rather than recomputing addresses. */ - strength_reduce(); - - /* Compact arenas after SSA optimization to free temporary SSA structures */ - compact_all_arenas(); - - /* Give each register-pair operation operands of its width. */ - widen_pair_operands(); - - /* SSA-based liveness analyses */ - liveness_analysis(); - - /* Compact after liveness analysis - mainly traversal args in GENERAL_ARENA - */ - compact_arenas_selective(COMPACT_ARENA_GENERAL); - - /* allocate register from IR */ - reg_alloc(); - - /* Compact after register allocation - mainly INSN and BB arenas */ - compact_arenas_selective(COMPACT_ARENA_INSN | COMPACT_ARENA_BB); - + mop_compute_liveness(); peephole(); - /* Apply arch-specific IR tweaks before final codegen */ - arch_lower(); + mop_compute_liveness(); /* flatten CFG to linear instruction */ cfg_flatten(); + dump_stats_phase("lower"); + vir_frontend_release(); /* Compact after CFG flattening - BB and GENERAL no longer needed */ compact_arenas_selective(COMPACT_ARENA_BB | COMPACT_ARENA_GENERAL); - /* dump second phase IR */ - if (dump_ir) - dump_ph2_ir(); + /* Lowering was the last reader of variables and scopes: the machine + * operations carry registers, offsets and interned names, and functions and + * types live in GENERAL_ARENA. + * + * Return them and every spare block before the backend and the ELF writer + * start allocating. + */ + release_block_arena(); /* ELF preprocess: * 1. generate all sections except for .text section. diff --git a/src/opt-sccp.c b/src/opt-sccp.c deleted file mode 100644 index d5ce04e6..00000000 --- a/src/opt-sccp.c +++ /dev/null @@ -1,98 +0,0 @@ -/* - * shecc - Self-Hosting and Educational C Compiler. - * - * shecc is freely redistributable under the BSD 2 clause license. See the file - * "LICENSE" for information on usage and redistribution of this file. - */ - -/* Constant cast optimization pass. - * - * Folds the 'load_constant' + 'trunc' pattern into a single constant load. - * General constant propagation and folding live in const_folding() in ssa.c, - * which runs over the same IR immediately afterwards. - */ - -/* Narrow a constant to 'size' bytes, keeping its sign. A signed narrow type - * sign-extends when widened, so masking alone would make "char c = -1" compare - * as 255. A size the caller does not narrow is returned unchanged. - */ -int sign_extend_const(int value, int size) -{ - if (size == 1) { - value = value & 0xFF; - if (value & 0x80) - value = value | ~0xFF; - } else if (size == 2) { - value = value & 0xFFFF; - if (value & 0x8000) - value = value | ~0xFFFF; - } - return value; -} - -/* Targeted constant truncation peephole optimization */ -bool optimize_constant_casts(func_t *func) -{ - if (!func || !func->bbs) - return false; - - bool changed = false; - - /* Simple peephole optimization: const + trunc pattern */ - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (insn_t *insn = bb->insn_list.head; insn && insn->next; - insn = insn->next) { - insn_t *next_insn = insn->next; - - /* Look for pattern: const %.tX, VALUE followed by %.tY = trunc - * %.tX, SIZE - */ - if (insn->opcode == OP_load_constant && - next_insn->opcode == OP_trunc && insn->rd && next_insn->rs1 && - insn->rd == next_insn->rs1 && next_insn->sz > 0 && - !next_insn->rd->is_global) { - int value = insn->rd->init_val; - int result = value; - - if (next_insn->sz != 1 && next_insn->sz != 2 && - next_insn->sz != 4) - continue; /* not a width we narrow to */ - result = sign_extend_const(value, next_insn->sz); - - /* An unsigned narrow type zero-extends instead: folding - * "unsigned short x = 0xfffe" to -2 made (int) x compare as -2. - */ - bool is_unsigned = next_insn->rd->type && - next_insn->rd->type->is_unsigned && - !next_insn->rd->ptr_level; - if (is_unsigned && next_insn->sz < 4) - result = value & ((1 << (next_insn->sz * 8)) - 1); - - /* Optimize: Replace both instructions with single const */ - insn->rd = next_insn->rd; /* Update dest to final target */ - insn->rd->is_const = true; - insn->rd->init_val = result; - - /* The high word belongs to the narrowed value too. A 64-bit - * target loads an argument constant at full width, and a zero - * high word made "f(0xe231fffab478ULL)" pass its int parameter - * as 0xfffab478 rather than -347016. - */ - insn->rd->init_val_hi = result < 0 && !is_unsigned ? -1 : 0; - - /* Remove the truncation instruction by converting it to - * NOP-like - */ - next_insn->opcode = OP_load_constant; - next_insn->rd->is_const = true; - next_insn->rd->init_val = result; - next_insn->rs1 = NULL; - next_insn->sz = 0; - - changed = true; - } - } - } - - return changed; -} diff --git a/src/parser-call.c b/src/parser-call.c index 8d759fac..7ee87f78 100644 --- a/src/parser-call.c +++ b/src/parser-call.c @@ -26,10 +26,7 @@ void read_func_parameters_with_sret(func_t *func, lex_expect(T_open_bracket); while (!lex_accept(T_close_bracket)) { - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } + read_assignment_or_expression(parent, bb); param = opstack_pop(); param = materialize_function_designator(parent, bb, param); @@ -69,13 +66,11 @@ void read_func_parameters_with_sret(func_t *func, if (!is_record_object(param)) error_at("Record argument required", cur_token_loc()); - var_t *copy = require_typed_var(parent, target->type); - copy->var_name = gen_name(); + var_t *copy = name_var(require_typed_var(parent, target->type)); add_insn(parent, *bb, OP_allocat, copy, NULL, NULL, 0, NULL); emit_record_copy(parent, bb, copy, param); - param = require_ref_var(parent, target->type, 0); - param->var_name = gen_name(); + param = name_var(require_ref_var(parent, target->type, 0)); add_insn(parent, *bb, OP_address_of, param, copy, NULL, 0, NULL); } else if (!has_effective_pointer(target) && !target->array_size) { @@ -96,12 +91,9 @@ void read_func_parameters_with_sret(func_t *func, if (!is_record_object(param)) error_at("Record argument required", cur_token_loc()); var_t *copy = require_typed_var(parent, param->type); - - copy->var_name = gen_name(); add_insn(parent, *bb, OP_allocat, copy, NULL, NULL, 0, NULL); emit_record_copy(parent, bb, copy, param); - param = require_ref_var(parent, param->type, 0); - param->var_name = gen_name(); + param = name_var(require_ref_var(parent, param->type, 0)); add_insn(parent, *bb, OP_address_of, param, copy, NULL, 0, NULL); } else if (!is_pointer_like_value(param) && @@ -203,6 +195,11 @@ void copy_call_result_array_shape(var_t *result, const var_t *return_def) : return_def->type->array_element_ptr_level); result->is_const_qualified = return_def->is_const_qualified || return_def->type->is_const_qualified; + result->pointer_const_mask = return_def->pointer_const_mask; + result->is_volatile = + return_def->is_volatile || return_def->type->is_volatile_qualified; + result->pointer_volatile_mask = return_def->pointer_volatile_mask | + return_def->type->pointer_volatile_mask; } /* The element type of array typedef @type, or @type itself when it is none. A @@ -233,17 +230,17 @@ static var_t *array_subscript_remainder(block_t *parent, for (int i = 0; i < consumed; i++) fixed_array_shape_drop_outer(&shape); if (unevaluated_expression_depth) { - result = require_typed_ptr_var(parent, type, element_ptr_level); + result = + name_var(require_typed_ptr_var(parent, type, element_ptr_level)); fixed_array_shape_to_var(result, &shape); - result->var_name = gen_name(); return result; } fixed_array_shape_drop_outer(&shape); - result = require_typed_ptr_var(parent, type, element_ptr_level + 1); + result = + name_var(require_typed_ptr_var(parent, type, element_ptr_level + 1)); fixed_array_shape_to_pointee_var(result, &shape); if (shape.rank) result->pointee_array_element_ptr_level = element_ptr_level; - result->var_name = gen_name(); add_insn(parent, *bb, OP_assign, result, address, NULL, 0, NULL); return result; } @@ -282,7 +279,6 @@ void lower_call_result_array_postfix(var_t **value, opstack_pop(); address = base; do { - var_t *index; int stride = element_ptr_level ? PTR_SIZE : base->type->size; if (depth >= dimensions) @@ -294,37 +290,14 @@ void lower_call_result_array_postfix(var_t **value, stride = fixed_array_shape_stride(&shape, depth - 1, stride); lex_expect(T_open_square); - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - index = opstack_pop(); - lex_expect(T_close_square); - if (stride != 1) { - var_t *scale = require_var(parent); - var_t *scaled = require_var(parent); - - scale->var_name = gen_name(); - scale->init_val = stride; - add_insn(parent, *bb, OP_load_constant, scale, NULL, NULL, 0, NULL); - scaled->var_name = gen_name(); - add_insn(parent, *bb, OP_mul, scaled, index, scale, 0, NULL); - index = scaled; - } - var_t *indexed = require_typed_ptr_var(parent, base->type, 1); - indexed->var_name = gen_name(); - add_insn(parent, *bb, OP_add, indexed, address, index, 0, NULL); - address = indexed; + address = + read_subscript_address(parent, bb, address, base->type, 1, stride); depth++; } while (lex_peek(T_open_square, NULL)); if (depth == dimensions) { - var_t *element = require_typed_var(parent, element_type); - opcode_t compound_op = OP_generic; - bool assignment; - bool record; - - element->ptr_level = element_ptr_level; + var_t *element = name_var( + require_typed_ptr_var(parent, element_type, element_ptr_level)); /* A call returning a pointer to an array of callbacks carries their * prototype on the result rather than on its scalar base type. @@ -334,101 +307,8 @@ void lower_call_result_array_postfix(var_t **value, : base->func_signature; element->is_const_qualified = base->is_const_qualified || base->type->is_const_qualified; - element->var_name = gen_name(); - - /* A record element is no register value: it stays at its address for a - * member selection or a record copy to read, as push_object_at() leaves - * one. - */ - record = is_record_object(element) && !element->func_signature && - !is_incomplete_record_object(element); - if (record) - element->defers_record_copy = true; - else - add_insn(parent, *bb, OP_read, element, address, NULL, - element->ptr_level ? PTR_SIZE : base->type->size, NULL); - element->is_compound_literal_reference = true; - element->compound_literal_address = address; - - assignment = - lex_accept(T_assign) || accept_compound_assign_op(&compound_op); - if (assignment) { - var_t *assigned; - - if (element->is_const_qualified) - error_at("assignment of read-only location", cur_token_loc()); - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - assigned = opstack_pop(); - if (record) { - if (compound_op != OP_generic || !is_record_object(assigned)) - error_at("Invalid record assignment", cur_token_loc()); - emit_record_copy_to_address(parent, bb, address, assigned); - *value = assigned; - opstack_push(*value); - return; - } - if (compound_op != OP_generic) { - if (is_pointer_operation(compound_op, element, assigned)) { - handle_pointer_arithmetic(parent, bb, compound_op, element, - assigned); - assigned = opstack_pop(); - } else { - var_t *current = - integer_promote_operand(parent, bb, element); - - assigned = integer_promote_operand(parent, bb, assigned); - normalize_integer_binary_operands(parent, bb, compound_op, - ¤t, &assigned); - var_t *combined = require_var(parent); - - combined->var_name = gen_name(); - combined->type = integer_binary_result_type( - compound_op, current, assigned); - add_insn(parent, *bb, compound_op, combined, current, - assigned, 0, NULL); - assigned = combined; - } - } - assigned = resize_var(parent, bb, assigned, element); - add_insn(parent, *bb, OP_write, NULL, address, assigned, - element->ptr_level ? PTR_SIZE : base->type->size, NULL); - *value = assigned; - opstack_push(*value); - return; - } - - if (lex_peek(T_increment, NULL) || lex_peek(T_decrement, NULL)) { - opcode_t op = lex_accept(T_increment) ? OP_add : OP_sub; - var_t *one; - var_t *updated; - - if (record) - error_at( - "Increment or decrement requires a scalar modifiable " - "lvalue", - cur_token_loc()); - if (element->is_const_qualified) - error_at("assignment of read-only location", cur_token_loc()); - one = require_typed_var(parent, TY_int); - one->var_name = gen_name(); - one->init_val = 1; - add_insn(parent, *bb, OP_load_constant, one, NULL, NULL, 0, NULL); - if (is_pointer_operation(op, element, one)) { - handle_pointer_arithmetic(parent, bb, op, element, one); - updated = opstack_pop(); - } else { - updated = require_var(parent); - updated->var_name = gen_name(); - updated->type = integer_binary_result_type(op, element, one); - add_insn(parent, *bb, op, updated, element, one, 0, NULL); - } - updated = resize_var(parent, bb, updated, element); - add_insn(parent, *bb, OP_write, NULL, address, updated, - element->ptr_level ? PTR_SIZE : base->type->size, NULL); - } + push_object_at(parent, bb, element, address, + element->ptr_level ? PTR_SIZE : base->type->size); *value = element; } else { /* The first subscript selected the pointed-to array itself. */ @@ -437,16 +317,7 @@ void lower_call_result_array_postfix(var_t **value, address); (*value)->is_const_qualified = base->is_const_qualified || base->type->is_const_qualified; - } - opstack_push(*value); - var_t *callable = *value; - if (callable->func_signature && lex_peek(T_open_bracket, NULL)) { - func_t *signature = callable->func_signature; - - var_t *result = - emit_indirect_call_result(callable, signature, true, parent, bb); - *value = result; - lower_call_result_array_postfix(value, parent, bb); + opstack_push(*value); } } @@ -457,7 +328,7 @@ void lower_call_result_prefix_update(var_t **value, { if (op == OP_generic) return; - *value = lower_reference_update(opstack_pop(), op, parent, bb); + *value = lower_reference_update(opstack_pop(), op, parent, bb, true); opstack_push(*value); } @@ -466,46 +337,11 @@ void lower_call_result_prefix_update(var_t **value, */ void read_builtin_offsetof(block_t *parent, basic_block_t **bb) { - char name[MAX_ID_LEN]; - type_t *record; - var_t *field; var_t *result; - int offset = 0; - bool has_nested_member; - - lex_expect(T_identifier); - lex_expect(T_open_bracket); - base_type_t record_kind = accept_record_keyword(); - if (record_kind) { - lex_ident(T_identifier, name); - record = find_record_tag(name, parent, record_kind); - } else { - lex_ident(T_identifier, name); - record = find_type(name, true); - } - if (!is_record_type(record)) - error_at("offsetof requires a struct or union type", cur_token_loc()); - lex_expect(T_comma); - do { - lex_ident(T_identifier, name); - field = find_member(name, record); - if (!field) - error_at("Unknown record member", cur_token_loc()); - offset += field->offset; - offset += read_offsetof_array_subscripts(field, parent); - record = field->type; - has_nested_member = lex_accept(T_dot); - if (has_nested_member && (field->ptr_level || !is_record_type(record))) - error_at("Nested offsetof member requires a record", - cur_token_loc()); - } while (has_nested_member); - lex_expect(T_close_bracket); - result = require_typed_var(parent, find_type("size_t", true)); - result->var_name = gen_name(); - result->init_val = offset; + result = load_constant(parent, *bb, read_offsetof_designator(parent), + find_builtin_type("size_t")); result->is_const = true; opstack_push(result); - add_insn(parent, *bb, OP_load_constant, result, NULL, NULL, 0, NULL); } /* Parse a C99 type name for the va_arg intrinsic. A type name has no @@ -551,9 +387,7 @@ void read_builtin_va_arg_type(block_t *parent, va_arg_type_t *result) ; while (lex_accept(T_asterisk)) { result->ptr_level++; - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; + skip_type_qualifiers(); } /* An array typedef is not itself an object type suitable for va_arg, but @@ -598,9 +432,7 @@ void read_builtin_va_arg_type(block_t *parent, va_arg_type_t *result) while (lex_accept(T_asterisk)) { nested_ptr++; - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; + skip_type_qualifiers(); } if (!nested_ptr) error_at("va_arg type name needs an abstract pointer declarator", @@ -611,8 +443,7 @@ void read_builtin_va_arg_type(block_t *parent, va_arg_type_t *result) while (lex_peek(T_open_square, NULL) || lex_peek(T_open_bracket, NULL)) { if (lex_accept(T_open_square)) { - int dims = 0; - int total = 0; + fixed_array_shape_t pointee_shape = {0}; if (type->base_type == TYPE_void) error_at( @@ -631,35 +462,15 @@ void read_builtin_va_arg_type(block_t *parent, va_arg_type_t *result) * used by normal declarations so the first subscript advances a * whole row (or plane), not one scalar element. */ - do { - int bound; - - if (dims >= 4) - error_at( - "Array type names support at most four dimensions", - cur_token_loc()); - if (lex_peek(T_close_square, NULL)) - error_at("pointer-to-array type needs a bound", - cur_token_loc()); - bound = read_const_expr(parent); - if (bound <= 0) - error_at("pointer-to-array bound must be positive", - cur_token_loc()); - lex_expect(T_close_square); - if (!dims) - total = bound; - else { - total *= bound; - if (dims == 1) - result->pointee_array_dim2 = bound; - else if (dims == 2) - result->pointee_array_dim3 = bound; - else - result->pointee_array_dim4 = bound; - } - dims++; - } while (lex_accept(T_open_square)); - result->pointee_array_size = total; + read_array_shape_bounds( + parent, &pointee_shape, + "Array type names support at most four dimensions", + "pointer-to-array type needs a bound", + "pointer-to-array bound must be positive", true, true); + fixed_array_shape_apply( + &pointee_shape, &result->pointee_array_size, + &result->pointee_array_dim2, &result->pointee_array_dim3, + &result->pointee_array_dim4); result->pointee_element_size = type->size; result->pointee_element_ptr_level = return_ptr_level + type->ptr_level; @@ -727,9 +538,7 @@ void emit_record_copy_from_address(block_t *parent, var_t *dest, var_t *src_addr) { - var_t *dest_addr = require_ref_var(parent, dest->type, 0); - - dest_addr->var_name = gen_name(); + var_t *dest_addr = name_var(require_ref_var(parent, dest->type, 0)); add_insn(parent, *bb, OP_address_of, dest_addr, dest, NULL, 0, NULL); emit_record_copy_between(parent, bb, dest_addr, src_addr, size_var(dest)); } @@ -786,14 +595,16 @@ void push_object_at(block_t *parent, } /* Apply ++ or -- (@op is OP_add or OP_sub) to @object, a value that records the - * address it was loaded from, and store the result there. + * address it was loaded from, and store the result there. Postfix callers can + * skip the bit-field reload when they only need the old value. * * Returns the value the object then holds. */ var_t *lower_reference_update(var_t *object, opcode_t op, block_t *parent, - basic_block_t **bb) + basic_block_t **bb, + bool need_updated_value) { var_t *one; var_t *updated; @@ -809,16 +620,12 @@ var_t *lower_reference_update(var_t *object, if (effective_pointer_depth(object) ? object->is_const_pointer : object->is_const_qualified) error_at("assignment of read-only location", cur_token_loc()); - one = require_typed_var(parent, TY_int); - one->var_name = gen_name(); - one->init_val = 1; - add_insn(parent, *bb, OP_load_constant, one, NULL, NULL, 0, NULL); + one = load_constant(parent, *bb, 1, TY_int); if (is_pointer_operation(op, object, one)) { handle_pointer_arithmetic(parent, bb, op, object, one); updated = opstack_pop(); } else { - updated = require_var(parent); - updated->var_name = gen_name(); + updated = require_named_var(parent); updated->type = integer_binary_result_type(op, object, one); add_insn(parent, *bb, op, updated, object, one, 0, NULL); } @@ -826,10 +633,12 @@ var_t *lower_reference_update(var_t *object, if (object->compound_literal_bitfield) { write_bitfield_value(parent, bb, object->compound_literal_address, updated, object->compound_literal_bitfield); - updated = - read_bitfield_value(parent, bb, object->compound_literal_address, - object->compound_literal_bitfield); - updated->is_bitfield = false; + if (need_updated_value) { + updated = read_bitfield_value(parent, bb, + object->compound_literal_address, + object->compound_literal_bitfield); + updated->is_bitfield = false; + } } else add_insn(parent, *bb, OP_write, NULL, object->compound_literal_address, updated, @@ -840,6 +649,36 @@ var_t *lower_reference_update(var_t *object, return updated; } +void read_builtin_va_start(block_t *parent, basic_block_t **bb) +{ + lex_expect(T_identifier); + lex_expect(T_open_bracket); + read_assignment_or_expression(parent, bb); + var_t *destination = opstack_pop(); + if (!has_effective_pointer(destination)) + error_at("va_start first argument must address a va_list", + cur_token_loc()); + lex_expect(T_comma); + char name[MAX_VAR_LEN]; + int parentheses = 0; + while (lex_accept(T_open_bracket)) + parentheses++; + lex_peek(T_identifier, name); + lex_expect(T_identifier); + while (parentheses--) + lex_expect(T_close_bracket); + func_t *func = parent->func; + if (!func || !func->va_args || !func->num_params || + find_var(name, parent) != &func->param_defs[func->num_params - 1]) + error_at("va_start requires the last named variadic parameter", + cur_token_loc()); + lex_expect(T_close_bracket); + var_t *value = name_var(require_typed_ptr_var(parent, TY_int, 1)); + add_insn(parent, *bb, OP_va_start, value, NULL, NULL, 0, NULL); + add_insn(parent, *bb, OP_write, NULL, destination, value, PTR_SIZE, NULL); + opstack_push(name_var(require_typed_var(parent, TY_void))); +} + void read_builtin_va_arg(block_t *parent, basic_block_t **bb) { va_arg_type_t requested; @@ -850,10 +689,7 @@ void read_builtin_va_arg(block_t *parent, basic_block_t **bb) lex_expect(T_identifier); lex_expect(T_open_bracket); - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } + read_assignment_or_expression(parent, bb); ap_addr = opstack_pop(); if (!has_effective_pointer(ap_addr)) error_at("va_arg first argument must be va_list lvalue", @@ -870,11 +706,9 @@ void read_builtin_va_arg(block_t *parent, basic_block_t **bb) if (requested.is_array || requested.is_function) error_at("va_arg type must be an object type", cur_token_loc()); - old = require_typed_ptr_var(parent, TY_int, 1); - old->var_name = gen_name(); + old = name_var(require_typed_ptr_var(parent, TY_int, 1)); add_insn(parent, *bb, OP_read, old, ap_addr, NULL, PTR_SIZE, NULL); - step = require_typed_var(parent, TY_int); - step->var_name = gen_name(); + step = name_var(require_typed_var(parent, TY_int)); /* A long long on a 32-bit target takes two ABI words starting at an even * one, both for AAPCS32 and for a variadic argument on RV32, and the callee @@ -883,23 +717,23 @@ void read_builtin_va_arg(block_t *parent, basic_block_t **bb) */ if (PTR_SIZE < 8 && !requested.ptr_level && !is_record_type(requested.type) && requested.type->size == 8) { - var_t *mask = require_typed_var(parent, TY_int); - var_t *aligned = require_typed_ptr_var(parent, TY_int, 1); + var_t *mask = name_var(require_typed_var(parent, TY_int)); + var_t *word = name_var(require_typed_var(parent, TY_uint)); + var_t *aligned = name_var(require_typed_var(parent, TY_uint)); + add_insn(parent, *bb, OP_cast, word, old, NULL, PTR_SIZE, NULL); step->init_val = 7; step->is_const = true; add_insn(parent, *bb, OP_load_constant, step, NULL, NULL, 0, NULL); - aligned->var_name = gen_name(); - add_insn(parent, *bb, OP_add, aligned, old, step, 0, NULL); - mask->var_name = gen_name(); + add_insn(parent, *bb, OP_add, aligned, word, step, 0, NULL); mask->init_val = -8; mask->is_const = true; add_insn(parent, *bb, OP_load_constant, mask, NULL, NULL, 0, NULL); - old = require_typed_ptr_var(parent, TY_int, 1); - old->var_name = gen_name(); - add_insn(parent, *bb, OP_bit_and, old, aligned, mask, 0, NULL); - step = require_typed_var(parent, TY_int); - step->var_name = gen_name(); + word = name_var(require_typed_var(parent, TY_uint)); + add_insn(parent, *bb, OP_bit_and, word, aligned, mask, 0, NULL); + old = name_var(require_typed_ptr_var(parent, TY_int, 1)); + add_insn(parent, *bb, OP_cast, old, word, NULL, PTR_SIZE, NULL); + step = name_var(require_typed_var(parent, TY_int)); } /* This direct IR add is byte-addressed; unlike parsed pointer arithmetic it @@ -912,26 +746,20 @@ void read_builtin_va_arg(block_t *parent, basic_block_t **bb) : PTR_SIZE; step->is_const = true; add_insn(parent, *bb, OP_load_constant, step, NULL, NULL, 0, NULL); - next = require_typed_ptr_var(parent, TY_int, 1); - next->var_name = gen_name(); + next = name_var(require_typed_ptr_var(parent, TY_int, 1)); add_insn(parent, *bb, OP_add, next, old, step, 0, NULL); add_insn(parent, *bb, OP_write, NULL, ap_addr, next, PTR_SIZE, NULL); if (is_record_type(requested.type) && !requested.ptr_level) { - var_t *source = require_ref_var(parent, requested.type, 0); - var_t *result = require_typed_var(parent, requested.type); - - source->var_name = gen_name(); + var_t *source = name_var(require_ref_var(parent, requested.type, 0)); + var_t *result = name_var(require_typed_var(parent, requested.type)); add_insn(parent, *bb, OP_read, source, old, NULL, PTR_SIZE, NULL); - result->var_name = gen_name(); add_insn(parent, *bb, OP_allocat, result, NULL, NULL, 0, NULL); emit_record_copy_from_address(parent, bb, result, source); opstack_push(result); } else { - var_t *result = - require_typed_ptr_var(parent, requested.type, requested.ptr_level); - - result->var_name = gen_name(); + var_t *result = name_var( + require_typed_ptr_var(parent, requested.type, requested.ptr_level)); add_insn(parent, *bb, OP_read, result, old, NULL, requested.ptr_level ? PTR_SIZE : requested.type->size, NULL); result->func_signature = requested.func_signature; @@ -961,9 +789,7 @@ void read_builtin_va_arg(block_t *parent, basic_block_t **bb) * pointer's width where an int was stored. */ type_t *element_type = - requested.pointee_element_ptr_level - ? pointee_type_from_pointer_typedef(requested.type) - : requested.type; + pointee_type_from_pointer_typedef(requested.type); while (lex_accept(T_open_square)) { var_t *index; @@ -991,24 +817,9 @@ void read_builtin_va_arg(block_t *parent, basic_block_t **bb) else for (int i = depth; i < dimension_count; i++) multiplier *= dims[i]; - if (multiplier != 1) { - var_t *scale = require_typed_var(parent, TY_int); - scale->var_name = gen_name(); - scale->init_val = multiplier; - scale->is_const = true; - add_insn(parent, *bb, OP_load_constant, scale, NULL, NULL, - 0, NULL); - var_t *scaled = require_var(parent); - scaled->var_name = gen_name(); - add_insn(parent, *bb, OP_mul, scaled, index, scale, 0, - NULL); - index = scaled; - } - address = require_typed_ptr_var( - parent, element_type, - requested.pointee_element_ptr_level + 1); - address->var_name = gen_name(); - add_insn(parent, *bb, OP_add, address, base, index, 0, NULL); + address = lower_array_element_address( + parent, *bb, base, index, element_type, + requested.pointee_element_ptr_level + 1, multiplier, true); depth++; if (depth == dimension_count + 1 && !requested.pointee_element_ptr_level && @@ -1017,18 +828,16 @@ void read_builtin_va_arg(block_t *parent, basic_block_t **bb) * read one scalar of the record's size, which drops bytes * and is a width no narrow backend can load. */ - var_t *value = require_typed_var(parent, requested.type); - - value->var_name = gen_name(); + var_t *value = + name_var(require_typed_var(parent, requested.type)); add_insn(parent, *bb, OP_allocat, value, NULL, NULL, 0, NULL); emit_record_copy_from_address(parent, bb, value, address); base = value; } else if (depth == dimension_count + 1) { - var_t *value = require_typed_ptr_var( + var_t *value = name_var(require_typed_ptr_var( parent, element_type, - requested.pointee_element_ptr_level); - value->var_name = gen_name(); + requested.pointee_element_ptr_level)); add_insn(parent, *bb, OP_read, value, address, NULL, element_size, NULL); base = value; @@ -1039,8 +848,8 @@ void read_builtin_va_arg(block_t *parent, basic_block_t **bb) } if (requested.func_signature && lex_peek(T_open_bracket, NULL)) { - result = emit_indirect_call_result(result, requested.func_signature, - true, parent, bb); + emit_indirect_call_result(result, requested.func_signature, true, + parent, bb); } } } @@ -1073,12 +882,10 @@ var_t *load_function_pointer_object(block_t *parent, for (int i = 0; i < parent->func->num_params; i++) { var_t *param = &parent->func->param_defs[i]; - if (object == param || object->base == param) { + if (object == param) { func_t *param_signature = get_func_signature(object); - var_t *value = require_typed_ptr_var( - parent, param_signature->return_def.type, 1); - - value->var_name = gen_name(); + var_t *value = name_var(require_typed_ptr_var( + parent, param_signature->return_def.type, 1)); value->func_signature = param_signature; add_insn(parent, *bb, OP_assign, value, object, NULL, 0, NULL); return value; @@ -1088,12 +895,9 @@ var_t *load_function_pointer_object(block_t *parent, var_t *address = require_ref_var(parent, object->type, object->ptr_level); func_t *signature = get_func_signature(object); - var_t *target = require_typed_ptr_var( - parent, signature ? signature->return_def.type : object->type, 1); - - address->var_name = gen_name(); + var_t *target = name_var(require_typed_ptr_var( + parent, signature ? signature->return_def.type : object->type, 1)); add_insn(parent, *bb, OP_address_of, address, object, NULL, 0, NULL); - target->var_name = gen_name(); target->func_signature = object->func_signature; add_insn(parent, *bb, OP_read, target, address, NULL, PTR_SIZE, NULL); return target; @@ -1125,9 +929,8 @@ var_t *materialize_function_designator(block_t *parent, if (!func) return value; - object = require_typed_ptr_var(parent, func->return_def.type, - func->return_def.ptr_level); - object->var_name = gen_name(); + object = name_var(require_typed_ptr_var(parent, func->return_def.type, + func->return_def.ptr_level)); object->is_func = true; object->func_signature = func; add_insn(parent, *bb, OP_allocat, object, NULL, NULL, 0, NULL); @@ -1152,6 +955,13 @@ void read_indirect_call_with_sret(var_t *callee, UNUSED(callee); + /* The evaluated temporary is the actual OP_indirect operand. Preserve the + * prototype on that value too, since later SSA lowering sees the operand + * rather than the source declaration that supplied the type. + */ + if (target && signature) + target->func_signature = signature; + read_func_parameters_with_sret(signature, sret, parent, bb); add_insn(parent, *bb, OP_indirect, NULL, target, NULL, 0, NULL); @@ -1172,14 +982,14 @@ var_t *prepare_aggregate_call_result(block_t *parent, func_t *signature, var_t **sret) { - var_t *result = require_typed_var(parent, signature->return_def.type); + var_t *result = + name_var(require_typed_var(parent, signature->return_def.type)); var_t *destination; - result->var_name = gen_name(); add_insn(parent, *bb, OP_allocat, result, NULL, NULL, 0, NULL); - destination = require_ref_var(parent, signature->return_def.type, 0); - destination->var_name = gen_name(); + destination = + name_var(require_ref_var(parent, signature->return_def.type, 0)); add_insn(parent, *bb, OP_address_of, destination, result, NULL, 0, NULL); *sret = destination; return result; @@ -1205,21 +1015,42 @@ static var_t *callback_slot_call_result(block_t *parent, if (slot && !return_def->type->array_size && !return_def->array_size) { result = require_typed_ptr_var(parent, return_def->type, return_def->ptr_level); - result->var_name = gen_name(); result->pointee_func_signature = slot; return result; } if (!callback || return_def->type->is_direct_function_type || return_def->type->array_size || return_def->ptr_level < 1) return NULL; - result = require_typed_ptr_var( + result = name_var(require_typed_ptr_var( parent, callback->return_def.type, - callback->return_def.ptr_level + return_def->ptr_level); - result->var_name = gen_name(); + callback->return_def.ptr_level + return_def->ptr_level)); result->pointee_func_signature = callback; return result; } +static var_t *emit_call_result_value(block_t *parent, + basic_block_t **bb, + const var_t *return_def, + bool direct_call) +{ + var_t *result = callback_slot_call_result(parent, return_def); + + if (!result) { + func_t *returned_signature = return_def->type->func_signature; + bool returns_function = direct_call && returned_signature; + + result = name_var(require_typed_ptr_var( + parent, + returns_function ? returned_signature->return_def.type + : return_def->type, + returns_function ? 1 : return_def->ptr_level)); + result->func_signature = returned_signature; + copy_call_result_array_shape(result, return_def); + } + add_insn(parent, *bb, OP_func_ret, result, NULL, NULL, 0, NULL); + return result; +} + /* Keep the ABI-only aggregate destination coupled to call emission. Call sites * that merely discard an aggregate result still must provide it. */ @@ -1228,7 +1059,6 @@ var_t *emit_direct_call_result(func_t *func, block_t *parent, basic_block_t **bb) { - func_t *returned_signature = func->return_def.type->func_signature; var_t *result = NULL; if (func->returns_aggregate) { @@ -1239,20 +1069,8 @@ var_t *emit_direct_call_result(func_t *func, } else { read_func_call(func, parent, bb); if (want_value) - result = callback_slot_call_result(parent, &func->return_def); - if (result) { - add_insn(parent, *bb, OP_func_ret, result, NULL, NULL, 0, NULL); - } else if (want_value) { - result = require_typed_ptr_var( - parent, - returned_signature ? returned_signature->return_def.type - : func->return_def.type, - returned_signature ? 1 : func->return_def.ptr_level); - result->var_name = gen_name(); - result->func_signature = returned_signature; - copy_call_result_array_shape(result, &func->return_def); - add_insn(parent, *bb, OP_func_ret, result, NULL, NULL, 0, NULL); - } + result = + emit_call_result_value(parent, bb, &func->return_def, true); } if (want_value) opstack_push(result); @@ -1282,17 +1100,8 @@ var_t *emit_indirect_call_result(var_t *callee, } else { read_indirect_call(callee, parent, bb); if (want_value && signature) - result = callback_slot_call_result(parent, &signature->return_def); - if (result) { - add_insn(parent, *bb, OP_func_ret, result, NULL, NULL, 0, NULL); - } else if (want_value && signature) { - result = require_typed_ptr_var(parent, signature->return_def.type, - signature->return_def.ptr_level); - result->var_name = gen_name(); - result->func_signature = signature->return_def.type->func_signature; - copy_call_result_array_shape(result, &signature->return_def); - add_insn(parent, *bb, OP_func_ret, result, NULL, NULL, 0, NULL); - } + result = emit_call_result_value(parent, bb, &signature->return_def, + false); } if (want_value) opstack_push(result); @@ -1301,12 +1110,8 @@ var_t *emit_indirect_call_result(var_t *callee, var_t *bitfield_constant(block_t *parent, basic_block_t **bb, unsigned value) { - var_t *result = require_var(parent); - result->var_name = gen_name(); - result->init_val = (int) value; + var_t *result = load_constant(parent, *bb, (int) value, TY_uint); result->is_const = true; - result->type = TY_uint; - add_insn(parent, *bb, OP_load_constant, result, NULL, NULL, 0, NULL); return result; } @@ -1317,8 +1122,7 @@ var_t *bitfield_binary(block_t *parent, var_t *right, type_t *type) { - var_t *result = require_var(parent); - result->var_name = gen_name(); + var_t *result = require_named_var(parent); result->type = type; add_insn(parent, *bb, op, result, left, right, 0, NULL); return result; @@ -1329,13 +1133,14 @@ unsigned bitfield_mask(const var_t *field) return field->bit_width == 32 ? ~0U : (1U << field->bit_width) - 1; } -var_t *read_bitfield_value(block_t *parent, - basic_block_t **bb, - var_t *address, - const var_t *field) +static var_t *read_bitfield_value_impl(block_t *parent, + basic_block_t **bb, + var_t *address, + const var_t *field, + bool assignment_reload) { - var_t *value = require_var(parent); - value->var_name = gen_name(); + var_t *value = require_named_var(parent); + value->is_assignment_reload = assignment_reload; value->type = TY_uint; add_insn(parent, *bb, OP_read, value, address, NULL, field->bit_storage_size, NULL); @@ -1381,6 +1186,14 @@ var_t *read_bitfield_value(block_t *parent, return value; } +var_t *read_bitfield_value(block_t *parent, + basic_block_t **bb, + var_t *address, + const var_t *field) +{ + return read_bitfield_value_impl(parent, bb, address, field, false); +} + /* The value of an assignment to a bit-field: the field read back from the unit * just written. C11 6.5.16p3 permits that read without requiring it, so mark it * removable when nothing uses the value; otherwise "s.flag = 1;" on a volatile @@ -1391,13 +1204,7 @@ var_t *reload_assigned_bitfield(block_t *parent, var_t *address, const var_t *field) { - insn_t *before = (*bb)->insn_list.tail; - basic_block_t *start = *bb; - var_t *value = read_bitfield_value(parent, bb, address, field); - - /* read_bitfield_value() emits the read first, into the given block. */ - insn_t *read = before ? before->next : start->insn_list.head; - read->rd->is_assignment_reload = true; + var_t *value = read_bitfield_value_impl(parent, bb, address, field, true); value->is_bitfield = false; return value; } @@ -1410,8 +1217,7 @@ void write_bitfield_value(block_t *parent, { if (is_bool_type(field->type)) value = normalize_bool(parent, bb, value); - var_t *old = require_var(parent); - old->var_name = gen_name(); + var_t *old = require_named_var(parent); old->type = TY_uint; add_insn(parent, *bb, OP_read, old, address, NULL, field->bit_storage_size, NULL); @@ -1423,8 +1229,7 @@ void write_bitfield_value(block_t *parent, parent, bb, OP_lshift, value, bitfield_constant(parent, bb, field->bit_offset), TY_uint); unsigned mask = bitfield_mask(field) << field->bit_offset; - var_t *clear = require_var(parent); - clear->var_name = gen_name(); + var_t *clear = require_named_var(parent); clear->type = TY_uint; add_insn(parent, *bb, OP_bit_not, clear, bitfield_constant(parent, bb, mask), NULL, 0, NULL); @@ -1468,9 +1273,8 @@ static void copy_pointee_array_shape(var_t *destination, const var_t *source) { fixed_array_shape_t shape = fixed_array_shape_from_pointee_var(source); - fixed_array_shape_to_pointee_var(destination, &shape); - destination->pointee_array_element_ptr_level = - source->pointee_array_element_ptr_level; + set_pointee_array_shape(destination, &shape, + source->pointee_array_element_ptr_level); } /* A dereference of a pointer to a pointer to an array, `*pp` for an int @@ -1525,19 +1329,13 @@ static void take_expression_address(block_t *parent, basic_block_t **bb) if (!effective_pointer_depth(operand) && !operand->array_size) error_at("Cannot dereference from non-pointer typed variable", cur_token_loc()); - result = require_typed_ptr_var( - parent, operand->type, operand->ptr_level + !!operand->array_size); - result->var_name = gen_name(); + result = name_var(require_typed_ptr_var( + parent, operand->type, operand->ptr_level + !!operand->array_size)); result->is_const_qualified = operand->is_const_qualified; result->pointer_const_mask = operand->pointer_const_mask; result->func_signature = operand->func_signature; result->pointee_func_signature = operand->pointee_func_signature; - result->pointee_array_size = operand->pointee_array_size; - result->pointee_array_dim2 = operand->pointee_array_dim2; - result->pointee_array_dim3 = operand->pointee_array_dim3; - result->pointee_array_dim4 = operand->pointee_array_dim4; - result->pointee_array_element_ptr_level = - operand->pointee_array_element_ptr_level; + copy_pointee_array_shape(result, operand); add_insn(parent, *bb, OP_assign, result, operand, NULL, 0, NULL); opstack_push(result); return; @@ -1564,9 +1362,8 @@ static void take_expression_address(block_t *parent, basic_block_t **bb) } else { error_at("Address-of requires an lvalue operand", cur_token_loc()); } - result = - require_typed_ptr_var(parent, operand->type, operand->ptr_level + 1); - result->var_name = gen_name(); + result = name_var( + require_typed_ptr_var(parent, operand->type, operand->ptr_level + 1)); result->is_const_qualified = operand->is_const_qualified; result->is_string_literal = operand->is_string_literal; if (operand->func_signature) @@ -1599,9 +1396,8 @@ void handle_address_of_operator(block_t *parent, basic_block_t **bb) error_at("__func__ is only defined inside a function", next_token_loc()); lex_expect(T_identifier); - vd = require_typed_ptr_var(parent, TY_char, 1); - vd->var_name = gen_name(); - vd->init_val = write_symbol(parent->func->return_def.var_name); + vd = name_var(require_typed_ptr_var(parent, TY_char, 1)); + vd->init_val = write_symbol(function_source_name(parent->func)); vd->is_string_literal = true; vd->is_func_name_array_address = true; opstack_push(vd); @@ -1623,9 +1419,7 @@ void handle_address_of_operator(block_t *parent, basic_block_t **bb) "function", next_token_loc()); lex_expect(T_identifier); - vd = require_func_symbol_var(parent); - vd->is_func = true; - vd->var_name = intern_string(token); + vd = function_designator_value(parent, addressed_func); opstack_push(vd); return; } @@ -1650,13 +1444,12 @@ void handle_address_of_operator(block_t *parent, basic_block_t **bb) fixed_array_shape_drop_outer(&shape); if (shape.rank) { rs1 = opstack_pop(); - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); vd->type = lvalue.decl->type; vd->ptr_level = lvalue.decl->ptr_level; vd->is_const_qualified = lvalue.decl->is_const_qualified; - copy_pointee_array_shape(vd, lvalue.decl); - fixed_array_shape_to_pointee_var(vd, &shape); + set_pointee_array_shape( + vd, &shape, lvalue.decl->pointee_array_element_ptr_level); add_insn(parent, *bb, OP_assign, vd, rs1, NULL, 0, NULL); opstack_push(vd); return; @@ -1665,8 +1458,7 @@ void handle_address_of_operator(block_t *parent, basic_block_t **bb) if (!lvalue.is_reference) { rs1 = opstack_pop(); - vd = require_ref_var(parent, lvalue.type, lvalue.ptr_level); - vd->var_name = gen_name(); + vd = name_var(require_ref_var(parent, lvalue.type, lvalue.ptr_level)); /* Address-of moves a pointer object's qualification one level inward: * `&p`, for `int * const p`, is `int * const *`, not `const int **`. @@ -1717,8 +1509,7 @@ void handle_address_of_operator(block_t *parent, basic_block_t **bb) for (int i = 0; i < lvalue.subscript_depth; i++) fixed_array_shape_drop_outer(&shape); - fixed_array_shape_to_pointee_var(vd, &shape); - vd->pointee_array_element_ptr_level = lvalue.decl->ptr_level; + set_pointee_array_shape(vd, &shape, lvalue.decl->ptr_level); } add_insn(parent, *bb, OP_assign, vd, rs1, NULL, 0, NULL); } else { @@ -1774,8 +1565,7 @@ static bool lower_pointee_array_dereference(var_t *source, */ if (!element_type->ptr_level) element_type = array_typedef_element_type(element_type); - row = require_var(parent); - row->type = element_type; + row = name_var(require_typed_var(parent, element_type)); fixed_array_shape_t shape = fixed_array_shape_from_pointee_var(source); fixed_array_shape_to_var(row, &shape); @@ -1793,12 +1583,12 @@ static bool lower_pointee_array_dereference(var_t *source, * lower levels of the pointer-to-array's mask belong to the element. */ if (row->ptr_level > 0 && row->ptr_level < 32) { - row->pointer_const_mask = - effective_pointer_const_mask(source) & ((1U << row->ptr_level) - 1); + unsigned int mask = (1U << row->ptr_level) - 1; + row->pointer_const_mask = effective_pointer_const_mask(source) & mask; + row->pointer_volatile_mask = source->pointer_volatile_mask & mask; row->is_const_pointer = row->pointer_const_mask & (1U << (row->ptr_level - 1)); } - row->var_name = gen_name(); add_insn(parent, *bb, OP_assign, row, source, NULL, 0, NULL); opstack_push(row); return true; @@ -1815,8 +1605,8 @@ static int pointee_array_row_stride(const var_t *var) return var->pointee_array_size * element_type->size; } -/* Byte extent remaining after direct-array subscript depth. This is shared by - * ordinary postfix indexing and the bounded grouped pointer-to-array path. +/* Byte extent remaining after an array subscript depth, shared by indexing and + * initializer layout. */ static int fixed_array_shape_stride(const fixed_array_shape_t *shape, int subscript_depth, @@ -1906,18 +1696,18 @@ static var_t *decay_dereferenced_array(block_t *parent, basic_block_t **bb, var_t *array) { - var_t *pointer = - require_typed_ptr_var(parent, array->type, array->ptr_level + 1); + var_t *pointer = name_var( + require_typed_ptr_var(parent, array->type, array->ptr_level + 1)); if (array->array_dim2) { fixed_array_shape_t shape = fixed_array_shape_from_var(array); fixed_array_shape_drop_outer(&shape); - fixed_array_shape_to_pointee_var(pointer, &shape); - pointer->pointee_array_element_ptr_level = array->ptr_level; + set_pointee_array_shape(pointer, &shape, array->ptr_level); } - pointer->var_name = gen_name(); pointer->is_const_qualified = array->is_const_qualified; + pointer->pointer_const_mask = array->pointer_const_mask; + pointer->pointer_volatile_mask = array->pointer_volatile_mask; add_insn(parent, *bb, OP_assign, pointer, array, NULL, 0, NULL); return pointer; } @@ -1931,6 +1721,23 @@ static bool dereferenced_call_follows(void) cur_token->next->next->kind == T_open_bracket; } +static void name_dereferenced_value(var_t *value, const var_t *source) +{ + value->var_name = gen_name(); + value->is_const_qualified = source->is_const_qualified; + value->is_volatile = source->is_volatile; + value->callback_is_volatile = source->callback_is_volatile; + int depth = source->ptr_level + source->type->ptr_level; + value->pointer_volatile_mask = + depth > 0 && depth <= 32 + ? source->pointer_volatile_mask & ~(1U << (depth - 1)) + : source->pointer_volatile_mask; + value->pointer_const_mask = dereferenced_pointer_const_mask(source); + value->is_const_pointer = + value->ptr_level > 0 && value->ptr_level <= 32 && + (value->pointer_const_mask & (1U << (value->ptr_level - 1))); +} + /* Dereference the pointer value @rs1 and push the object it designates. */ void push_dereference(block_t *parent, basic_block_t **bb, var_t *rs1) { @@ -1944,8 +1751,7 @@ void push_dereference(block_t *parent, basic_block_t **bb, var_t *rs1) * element. Dereferencing it restores the array, which immediately * decays back to char * in this expression context. */ - vd = require_typed_ptr_var(parent, TY_char, 1); - vd->var_name = gen_name(); + vd = name_var(require_typed_ptr_var(parent, TY_char, 1)); vd->is_string_literal = true; opstack_push(vd); add_insn(parent, *bb, OP_assign, vd, rs1, NULL, 0, NULL); @@ -1981,13 +1787,8 @@ void push_dereference(block_t *parent, basic_block_t **bb, var_t *rs1) ? PTR_SIZE : pointer_typedef_pointee_size( deref_type, pointee_type_from_pointer_typedef(deref_type)); - vd->var_name = gen_name(); - vd->is_const_qualified = rs1->is_const_qualified; + name_dereferenced_value(vd, rs1); vd->is_string_literal = rs1->is_string_literal; - vd->pointer_const_mask = dereferenced_pointer_const_mask(rs1); - vd->is_const_pointer = - vd->ptr_level > 0 && vd->ptr_level <= 32 && - (vd->pointer_const_mask & (1U << (vd->ptr_level - 1))); /* Pointer arithmetic can produce a pointer to a callback typedef. The * actual dereference consumes that object-pointer level and yields the @@ -2047,12 +1848,9 @@ static bool push_dereferenced_function(block_t *parent, /* An array of function pointers decays to the address of its first * element, from which `*fps` reads the pointer, as fps[0] does. */ - var_t *address = require_typed_ptr_var(parent, rs1->type, 1); - var_t *target = require_typed_ptr_var(parent, rs1->type, 1); - - address->var_name = gen_name(); + var_t *address = name_var(require_typed_ptr_var(parent, rs1->type, 1)); + var_t *target = name_var(require_typed_ptr_var(parent, rs1->type, 1)); add_insn(parent, *bb, OP_assign, address, rs1, NULL, 0, NULL); - target->var_name = gen_name(); target->func_signature = rs1->func_signature; add_insn(parent, *bb, OP_read, target, address, NULL, PTR_SIZE, NULL); opstack_push(target); @@ -2065,11 +1863,63 @@ static bool push_dereferenced_function(block_t *parent, return false; } -void handle_single_dereference(block_t *parent, basic_block_t **bb) +static bool restore_dereferenced_callback(var_t *value, + var_t *source, + type_t *deref_type, + int *size); +static bool push_dereferenced_object(block_t *parent, + basic_block_t **bb, + var_t *source); + +static void handle_simple_dereferences(block_t *parent, + basic_block_t **bb, + int count) { - var_t *vd, *rs1; - int sz; + char token[MAX_VAR_LEN]; + lvalue_t lvalue; + var_t *var; + var_t *value; + + if (!lex_peek(T_identifier, token)) + error_at("Expected an identifier", next_token_loc()); + var = find_var(token, parent); + + /* A raw function name is a designator, not an object lvalue. Keep the + * single-star symbol case on its direct path; the multi-star parser + * historically requires an object declaration here. + */ + func_t *func = count == 1 && !var ? find_visible_func(token, parent) : NULL; + if (func) { + lex_expect(T_identifier); + value = function_designator_value(parent, func); + opstack_push(value); + return; + } + + read_lvalue(&lvalue, var, parent, bb, true, OP_generic); + + for (int i = 0; i < count; i++) { + value = opstack_pop(); + if (i == 0 && !decays_when_dereferenced(value) && + push_dereferenced_function(parent, bb, &lvalue, var, value)) + break; + if (decays_when_dereferenced(value)) { + push_dereference(parent, bb, value); + } else if (lower_pointee_array_dereference(value, parent, bb)) { + continue; + } else if (count == 1 && value->func_signature && + !effective_pointer_depth(value)) { + value->ptr_level = 1; + opstack_push(value); + break; + } else if (push_dereferenced_object(parent, bb, value)) { + break; + } + } +} +void handle_single_dereference(block_t *parent, basic_block_t **bb) +{ if (lex_peek(T_open_bracket, NULL) || lex_peek(T_increment, NULL) || lex_peek(T_decrement, NULL) || lex_peek(T_ampersand, NULL) || dereferenced_call_follows()) { @@ -2082,120 +1932,55 @@ void handle_single_dereference(block_t *parent, basic_block_t **bb) read_expr_operand(parent, bb); push_dereference(parent, bb, opstack_pop()); } else { - /* Handle simple identifier dereference: *var */ - char token[MAX_VAR_LEN]; - lvalue_t lvalue; - - if (!lex_peek(T_identifier, token)) - error_at("Expected an identifier", next_token_loc()); - - /* Builtins are expression operands, not objects in the local symbol - * table. Keep the identifier lvalue path below for `*pointer` - * assignments, but let a pointer-valued builtin such as `*va_arg(...)` - * use the same rvalue dereference lowering as `*(expression)`. - */ - if (!strcmp(token, "__builtin_va_arg")) { - type_t *deref_type; - int deref_ptr; - - read_expr_operand(parent, bb); - rs1 = opstack_pop(); - deref_type = rs1->type ? rs1->type : TY_int; - deref_ptr = rs1->ptr_level + deref_type->ptr_level - 1; - vd = require_deref_var(parent, deref_type, rs1->ptr_level); - sz = deref_ptr > 0 ? PTR_SIZE : deref_type->size; - vd->var_name = gen_name(); - vd->is_const_qualified = rs1->is_const_qualified; - vd->pointer_const_mask = dereferenced_pointer_const_mask(rs1); - vd->is_const_pointer = - vd->ptr_level > 0 && vd->ptr_level <= 32 && - (vd->pointer_const_mask & (1U << (vd->ptr_level - 1))); - push_object_at(parent, bb, vd, rs1, sz); - return; - } - var_t *var = find_var(token, parent); - - /* A raw function name is a function designator, not an object lvalue. - * Dereferencing it is a no-op in C99 (`*f` is another designator), so - * route it through the same symbol representation used by ordinary - * direct calls instead of asking read_lvalue() to find object storage. - */ - if (!var && find_visible_func(token, parent)) { - lex_expect(T_identifier); - rs1 = require_func_symbol_var(parent); - rs1->var_name = intern_string(token); - rs1->is_func = true; - opstack_push(rs1); - return; - } - read_lvalue(&lvalue, var, parent, bb, true, OP_generic); - - rs1 = opstack_pop(); - if (decays_when_dereferenced(rs1)) { - push_dereference(parent, bb, rs1); - return; - } - if (push_dereferenced_function(parent, bb, &lvalue, var, rs1)) - return; - if (lower_pointee_array_dereference(rs1, parent, bb)) - return; - - /* A member function-pointer typedef was already loaded by - * read_lvalue(). Unary `*` turns that pointer into a function - * designator; it does not read from the code address. - */ - if (rs1->func_signature && !effective_pointer_depth(rs1)) { - rs1->ptr_level = 1; - opstack_push(rs1); - return; - } + handle_simple_dereferences(parent, bb, 1); + } +} - /* `read_lvalue()` may have resolved a member expression. Derive the - * final indirection from its evaluated value rather than from the - * initial record object, so `*record.pointer` dereferences the member - * pointer instead of attempting to dereference the record itself. - */ - type_t *deref_type = rs1->type ? rs1->type : TY_int; - int deref_ptr = rs1->ptr_level + deref_type->ptr_level - 1; - - vd = require_deref_var(parent, deref_type, rs1->ptr_level); - sz = deref_ptr > 0 ? PTR_SIZE : deref_type->size; - vd->var_name = gen_name(); - vd->is_const_qualified = rs1->is_const_qualified; - vd->pointer_const_mask = dereferenced_pointer_const_mask(rs1); - vd->is_const_pointer = - vd->ptr_level > 0 && vd->ptr_level <= 32 && - (vd->pointer_const_mask & (1U << (vd->ptr_level - 1))); - - /* `*slot` is the function-pointer value when slot is a pointer to a - * callback typedef. The outer declarator's signature was deliberately - * cleared to prevent the invalid `slot(...)` form, so restore the - * element signature only after the real dereference has occurred. - */ - if (rs1->pointee_func_signature - ? callback_slot_depth(rs1) == 1 - : ((rs1->func_signature || - (deref_type && deref_type->func_signature)) && - effective_pointer_depth(rs1) == 1)) { - vd->func_signature = - rs1->pointee_func_signature - ? rs1->pointee_func_signature - : (rs1->func_signature ? rs1->func_signature - : deref_type->func_signature); - vd->ptr_level = 1; - sz = PTR_SIZE; - } - keep_callback_slot(vd, rs1); - keep_pointee_array_shape(vd, rs1); - push_object_at(parent, bb, vd, rs1, sz); +static bool restore_dereferenced_callback(var_t *value, + var_t *source, + type_t *deref_type, + int *size) +{ + if (source->pointee_func_signature + ? callback_slot_depth(source) == 1 + : ((source->func_signature || + (deref_type && deref_type->func_signature)) && + effective_pointer_depth(source) == 1)) { + value->func_signature = + source->pointee_func_signature + ? source->pointee_func_signature + : (source->func_signature ? source->func_signature + : deref_type->func_signature); + value->ptr_level = 1; + *size = PTR_SIZE; + return true; } + return false; } -void handle_multiple_dereference(block_t *parent, basic_block_t **bb) +static bool push_dereferenced_object(block_t *parent, + basic_block_t **bb, + var_t *source) { - var_t *vd, *rs1; - int sz; + type_t *type = source->type ? source->type : TY_int; + int pointer_level = source->ptr_level + type->ptr_level - 1; + int size = pointer_level > 0 + ? PTR_SIZE + : pointer_typedef_pointee_size( + type, pointee_type_from_pointer_typedef(type)); + var_t *value = require_deref_var(parent, type, source->ptr_level); + bool is_callback; + + name_dereferenced_value(value, source); + is_callback = restore_dereferenced_callback(value, source, type, &size); + keep_callback_slot(value, source); + keep_pointee_array_shape(value, source); + push_object_at(parent, bb, value, source, size); + return is_callback; +} +void handle_multiple_dereference(block_t *parent, basic_block_t **bb) +{ /* Handle consecutive asterisks for multiple dereference: **pp, ***ppp, and * the parenthesized ***(expr) form. */ @@ -2213,79 +1998,7 @@ void handle_multiple_dereference(block_t *parent, basic_block_t **bb) for (int i = 0; i < deref_count; i++) push_dereference(parent, bb, opstack_pop()); } else { - /* Handle **pp, ***ppp case with simple identifier */ - char token[MAX_VAR_LEN]; - lvalue_t lvalue; - - if (!lex_peek(T_identifier, token)) - error_at("Expected an identifier", next_token_loc()); - var_t *var = find_var(token, parent); - read_lvalue(&lvalue, var, parent, bb, true, OP_generic); - - /* Apply dereferences one by one. Once they reach a function, the - * remaining ones are no-ops: `**fp` designates the function `*fp` does. - */ - bool designator = false; - for (int i = 0; i < deref_count; i++) { - rs1 = opstack_pop(); - if (designator) { - opstack_push(rs1); - continue; - } - if (i == 0 && !decays_when_dereferenced(rs1) && - push_dereferenced_function(parent, bb, &lvalue, var, rs1)) { - designator = true; - continue; - } - if (decays_when_dereferenced(rs1)) { - push_dereference(parent, bb, rs1); - continue; - } - if (lower_pointee_array_dereference(rs1, parent, bb)) - continue; - - /* A member lvalue has already been read into rs1. Each unary - * asterisk must consume that evaluated pointer, not re-derive its - * type from the initial record identifier. - */ - type_t *deref_type = rs1->type ? rs1->type : TY_int; - int deref_ptr = rs1->ptr_level + deref_type->ptr_level - 1; - - vd = require_deref_var(parent, deref_type, rs1->ptr_level); - sz = deref_ptr > 0 - ? PTR_SIZE - : pointer_typedef_pointee_size( - deref_type, - pointee_type_from_pointer_typedef(deref_type)); - vd->var_name = gen_name(); - vd->is_const_qualified = rs1->is_const_qualified; - vd->pointer_const_mask = dereferenced_pointer_const_mask(rs1); - vd->is_const_pointer = - vd->ptr_level > 0 && vd->ptr_level <= 32 && - (vd->pointer_const_mask & (1U << (vd->ptr_level - 1))); - - /* Only the dereference of a pointer to a callback, the last read of - * `**slots` or `**fpp`, reaches the callback object. Earlier reads - * still produce a pointer-to-callback and must not be callable. - */ - if (rs1->pointee_func_signature - ? callback_slot_depth(rs1) == 1 - : ((rs1->func_signature || - (deref_type && deref_type->func_signature)) && - effective_pointer_depth(rs1) == 1)) { - vd->func_signature = - rs1->pointee_func_signature - ? rs1->pointee_func_signature - : (rs1->func_signature ? rs1->func_signature - : deref_type->func_signature); - vd->ptr_level = 1; - sz = PTR_SIZE; - designator = true; - } - keep_callback_slot(vd, rs1); - keep_pointee_array_shape(vd, rs1); - push_object_at(parent, bb, vd, rs1, sz); - } + handle_simple_dereferences(parent, bb, deref_count); } } @@ -2324,30 +2037,9 @@ void lower_member_postfix(var_t *address, ? PTR_SIZE : field->type->size; int stride = fixed_array_shape_stride(&shape, depth, element_size); - var_t *index; - - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - index = opstack_pop(); - lex_expect(T_close_square); - if (stride != 1) { - var_t *scale = require_var(parent); - var_t *scaled = require_var(parent); - scale->var_name = gen_name(); - scale->init_val = stride; - add_insn(parent, *bb, OP_load_constant, scale, NULL, NULL, 0, - NULL); - scaled->var_name = gen_name(); - add_insn(parent, *bb, OP_mul, scaled, index, scale, 0, NULL); - index = scaled; - } - var_t *indexed = require_typed_ptr_var(parent, field->type, 1); - indexed->var_name = gen_name(); - add_insn(parent, *bb, OP_add, indexed, address, index, 0, NULL); - address = indexed; + address = read_subscript_address(parent, bb, address, field->type, + 1, stride); depth++; if (depth == shape.rank && is_record_type(field->type) && !field->ptr_level) @@ -2374,10 +2066,8 @@ void lower_member_postfix(var_t *address, if (depth < shape.rank || effective_pointer_depth(field) != 1 || field->is_func || !is_record_type(pointee)) error_at("Invalid record member access", cur_token_loc()); - var_t *pointer = - require_typed_ptr_var(parent, field->type, field->ptr_level); - - pointer->var_name = gen_name(); + var_t *pointer = name_var( + require_typed_ptr_var(parent, field->type, field->ptr_level)); add_insn(parent, *bb, OP_read, pointer, address, NULL, PTR_SIZE, NULL); address = pointer; @@ -2387,9 +2077,8 @@ void lower_member_postfix(var_t *address, field->is_const_qualified || field->type->is_const_qualified; } else { lex_expect(T_dot); - if (field) - record_const = record_const || field->is_const_qualified || - field->type->is_const_qualified; + record_const = record_const || field->is_const_qualified || + field->type->is_const_qualified; } if (!record_type) error_at("Member access requires a record", cur_token_loc()); @@ -2412,18 +2101,16 @@ void lower_member_postfix(var_t *address, /* sizeof observes the array itself, before any decay. */ for (int i = 0; i < depth; i++) fixed_array_shape_drop_outer(&shape); - result = require_typed_ptr_var(parent, field->type, field->ptr_level); + result = name_var( + require_typed_ptr_var(parent, field->type, field->ptr_level)); fixed_array_shape_to_var(result, &shape); - result->var_name = gen_name(); opstack_push(result); } else if (decayed) { for (int i = 0; i <= depth; i++) fixed_array_shape_drop_outer(&shape); - result = - require_typed_ptr_var(parent, field->type, field->ptr_level + 1); - fixed_array_shape_to_pointee_var(result, &shape); - result->pointee_array_element_ptr_level = field->ptr_level; - result->var_name = gen_name(); + result = name_var( + require_typed_ptr_var(parent, field->type, field->ptr_level + 1)); + set_pointee_array_shape(result, &shape, field->ptr_level); add_insn(parent, *bb, OP_assign, result, address, NULL, 0, NULL); opstack_push(result); } else if (is_bitfield(field)) { @@ -2432,8 +2119,8 @@ void lower_member_postfix(var_t *address, if (is_lvalue) mark_value_reference(result, address, field); } else { - result = require_typed_ptr_var(parent, field->type, field->ptr_level); - result->var_name = gen_name(); + result = name_var( + require_typed_ptr_var(parent, field->type, field->ptr_level)); result->func_signature = field->func_signature; /* A function-pointer member is loaded as the pointer value a call diff --git a/src/parser-const.c b/src/parser-const.c index 25bafd69..3a1cffbe 100644 --- a/src/parser-const.c +++ b/src/parser-const.c @@ -29,7 +29,7 @@ static int read_global_sizeof_expression(block_t *scope) int read_const_wstring_size(void) { int values[MAX_STRING_LEN]; - type_t *wide_type = find_type("wchar_t", true); + type_t *wide_type = find_builtin_type("wchar_t"); return (read_wstring_units(values, MAX_STRING_LEN) + 1) * wide_type->size; } @@ -53,154 +53,52 @@ int read_sizeof_constant(block_t *scope) int eval_expression_imm(opcode_t op, int op1, int op2) { - /* return immediate result */ - int tmp = op2; int res = 0; - switch (op) { - case OP_add: - if (checking_enum_constant && ((op2 > 0 && op1 > INT_MAX - op2) || - (op2 < 0 && op1 < INT_MIN - op2))) + + if (checking_enum_constant) { + if (op == OP_add && ((op2 > 0 && op1 > INT_MAX - op2) || + (op2 < 0 && op1 < INT_MIN - op2))) error_at("Enumerator value exceeds int range", cur_token_loc()); - res = op1 + op2; - break; - case OP_sub: - if (checking_enum_constant && ((op2 < 0 && op1 > INT_MAX + op2) || - (op2 > 0 && op1 < INT_MIN + op2))) + if (op == OP_sub && ((op2 < 0 && op1 > INT_MAX + op2) || + (op2 > 0 && op1 < INT_MIN + op2))) error_at("Enumerator value exceeds int range", cur_token_loc()); - res = op1 - op2; - break; - case OP_mul: - if (checking_enum_constant && op1 && op2 && + if (op == OP_mul && op1 && op2 && ((op1 > 0 && op2 > 0 && op1 > INT_MAX / op2) || (op1 > 0 && op2 < 0 && op2 < INT_MIN / op1) || (op1 < 0 && op2 > 0 && op1 < INT_MIN / op2) || (op1 < 0 && op2 < 0 && op1 < INT_MAX / op2))) error_at("Enumerator value exceeds int range", cur_token_loc()); - res = op1 * op2; - break; - case OP_div: + if (op == OP_lshift && + (op2 < 0 || op2 >= 32 || op1 < 0 || op1 > (INT_MAX >> op2))) + error_at("Enumerator value exceeds int range", cur_token_loc()); + } + if (op == OP_div || op == OP_mod) { if (!op2) - error_at("Division by zero in constant expression", + error_at(op == OP_div ? "Division by zero in constant expression" + : "Modulo by zero in constant expression", cur_token_loc()); - - /* INT_MIN / -1 has no representable result; on x86 it raises SIGFPE - * rather than producing one. - */ if (op1 == INT_MIN && op2 == -1) error_at("Overflow in constant expression", cur_token_loc()); - res = op1 / op2; - break; - case OP_mod: - if (!op2) - error_at("Modulo by zero in constant expression", cur_token_loc()); - if (op1 == INT_MIN && op2 == -1) - error_at("Overflow in constant expression", cur_token_loc()); - /* Use bitwise AND for modulo optimization when divisor is power of 2 */ - if (tmp == INT_MIN) { - res = op1 % op2; - break; - } - tmp = tmp < 0 ? -tmp : tmp; - tmp &= (tmp - 1); - if (tmp != 0) { - res = op1 % op2; - break; - } - op2 = op2 < 0 ? -op2 : op2; - res = op1 & (op2 - 1); - if (op1 < 0 && res != 0) - res -= op2; - break; - case OP_lshift: - if (checking_enum_constant && - (op2 < 0 || op2 >= 32 || op1 < 0 || op1 > (INT_MAX >> op2))) - error_at("Enumerator value exceeds int range", cur_token_loc()); - - /* A count outside the int width has no defined result, and the host - * compiler must not be asked to produce one. Shift the unsigned bit - * pattern so a set sign bit is not host undefined behavior either. - */ - if (op2 < 0 || op2 >= 32) - error_at("Shift count out of range in constant expression", - cur_token_loc()); - res = (int) ((unsigned int) op1 << op2); - break; - case OP_rshift: - if (op2 < 0 || op2 >= 32) - error_at("Shift count out of range in constant expression", - cur_token_loc()); - res = op1 >> op2; - break; - case OP_log_and: - res = op1 && op2; - break; - case OP_log_or: - res = op1 || op2; - break; - case OP_eq: - res = op1 == op2; - break; - case OP_neq: - res = op1 != op2; - break; - case OP_lt: - res = op1 < op2; - break; - case OP_gt: - res = op1 > op2; - break; - case OP_leq: - res = op1 <= op2; - break; - case OP_geq: - res = op1 >= op2; - break; - case OP_bit_and: - res = op1 & op2; - break; - case OP_bit_or: - res = op1 | op2; - break; - case OP_bit_xor: - res = op1 ^ op2; - break; - default: - error_at("The requested operation is not supported.", cur_token_loc()); } + if ((op == OP_lshift || op == OP_rshift) && (op2 < 0 || op2 >= 32)) + error_at("Shift count out of range in constant expression", + cur_token_loc()); + if (!vir_frontend_fold_i32(op, op1, op2, &res)) + error_at("The requested operation is not supported.", cur_token_loc()); return res; } bool read_global_assignment_var(var_t *var); -/* Diagnose the conversion of the function designator @symbol to @var, or of the - * enclosing cast to a function pointer type when there is one. - */ -static void diagnose_global_function_conversion(var_t *symbol, var_t *var) -{ - var_t cast = {0}; - - if (!global_function_cast_signature) { - diagnose_function_pointer_conversion(symbol, var); - return; - } - cast.type = global_function_cast_signature->return_def.type; - cast.ptr_level = 1; - cast.func_signature = global_function_cast_signature; - diagnose_function_pointer_conversion(&cast, var); -} - /* The integer evaluators below yield constants only, so a nonzero value for a * pointer object, a function pointer among them, is an integer converted - * without a cast. Under a cast to a function pointer type, the conversion is - * the explicit one the cast performs. + * without a cast. Explicit pointer casts are retained on the source value. */ static void reject_global_integer_pointer(const var_t *dest, const var_t *src) { if ((effective_pointer_depth(dest) || dest->is_func) && !dest->array_size) { - if (global_function_cast_signature) - diagnose_global_function_conversion((var_t *) src, (var_t *) dest); - else if (!is_pointer_like_value((var_t *) src) && !src->is_func && - !src->is_string_literal && (src->init_val || src->init_val_hi)) + if (!is_pointer_like_value((var_t *) src) && !src->is_func && + !src->is_string_literal && (src->init_val || src->init_val_hi)) error_at("integer converted to pointer without a cast", cur_token_loc()); } @@ -258,22 +156,14 @@ static bool initializer_needs_wide_reader(token_t *token, block_t *scope, bool casts) { - int bracket_depth = 0; + int depth = 0; for (; token; token = token->next) { if ((token->kind == T_numeric && - (numeric_literal_needs_wide_path(token->literal) || - numeric_literal_needs_typed_global_path(token->literal))) || + numeric_literal_needs_global_reader(token->literal)) || (casts && global_integer_cast_starts_at(token, scope))) return true; - if (token->kind == T_open_bracket) - bracket_depth++; - else if (token->kind == T_close_bracket) { - if (bracket_depth == 0) - return false; - bracket_depth--; - } else if (bracket_depth == 0 && - (token->kind == T_semicolon || token->kind == T_comma)) + if (!initializer_scan_continue(token, &depth, false)) return false; } return false; @@ -290,32 +180,6 @@ bool typed_global_literal_appears_before_initializer_end(token_t *token) return initializer_needs_wide_reader(token, NULL, false); } -/* Whether a subscripted string literal, as in `"ab"[1]`, appears in the static - * initializer that starts at @token. Only the literal expression reader folds - * it; integer constant expressions such as enumerators do not admit it. - */ -bool string_element_appears_before_initializer_end(token_t *token) -{ - int bracket_depth = 0; - - for (; token; token = token->next) { - if ((token->kind == T_string || token->kind == T_wstring) && - token->next && token->next->kind == T_open_square) - return true; - if (token->kind == T_open_bracket) - bracket_depth++; - else if (token->kind == T_close_bracket) { - if (bracket_depth == 0) - return false; - bracket_depth--; - } else if (bracket_depth == 0 && - (token->kind == T_semicolon || token->kind == T_comma || - token->kind == T_close_curly)) - return false; - } - return false; -} - var_t *read_wide_global_literal_expression(block_t *parent, basic_block_t *bb, block_t *scope); @@ -335,34 +199,6 @@ int narrow_wide_global_address_offset(var_t *value) return value->init_val; } -/* A global pointer offset may use the same literal-only wide expression as a - * scalar global initializer. Its final index still flows through the current - * word-sized address relocation representation. - */ -int read_global_address_offset(block_t *scope, - block_t *parent, - basic_block_t *bb) -{ - token_t *operator_token = cur_token->next; - - if (operator_token && operator_token->next && - typed_global_literal_appears_before_initializer_end( - operator_token->next)) { - bool negate = lex_accept(T_minus); - var_t *wide_offset; - int index; - - if (!negate) - lex_expect(T_plus); - wide_offset = read_wide_global_literal_expression(parent, bb, scope); - index = narrow_wide_global_address_offset(wide_offset); - return negate ? -index : index; - } - return read_const_expr(scope); -} - -static int wide_global_unevaluated_depth; - /* The high word an int-sized value of this type has once it is widened: a * signed source sign-extends and an unsigned source zero-extends. */ @@ -397,7 +233,9 @@ static void store_wide_global_word_result(block_t *parent, unsigned int lo, unsigned int hi) { - if (result->type && result->type->size <= TY_int->size) + if (is_pointer_like_value(result)) + hi = PTR_SIZE == 8 ? hi : 0; + else if (result->type && result->type->size <= TY_int->size) hi = wide_global_narrow_high(lo, result->type->is_unsigned); result->init_val = lo; result->init_val_hi = hi; @@ -429,38 +267,59 @@ bool emit_wide_global_word_arithmetic(block_t *parent, return true; } + if ((op == OP_lshift || op == OP_rshift) && + (rhs_hi || rhs_lo >= (left->type->size <= TY_int->size ? 32u : 64u))) + error_at("Shift count out of range in constant expression", + cur_token_loc()); + /* An address constant plus or minus an integer constant is itself an * address constant (C99 6.6p7), with the integer first in a sum as well. */ if ((op == OP_add || op == OP_sub) && right) { - var_t *address = wide_global_address_operand(left) ? left - : op == OP_add ? right - : NULL; + var_t *address = + (wide_global_address_operand(left) || is_pointer_like_value(left)) + ? left + : op == OP_add ? right + : NULL; var_t *index = address == left ? right : left; int stride = !address || address->is_func ? 0 + : address->address_stride ? address->address_stride : address->is_string_literal ? address->type->size - : address->address_stride; + : 0; - if (stride && wide_global_address_operand(address) && - !wide_global_address_operand(index)) { - var_t *offset = require_var(parent); + if (stride && + (wide_global_address_operand(address) || + is_pointer_like_value(address)) && + !wide_global_address_operand(index) && + !is_pointer_like_value(index)) { int step = narrow_wide_global_address_offset(index); if (step > INT_MAX / stride || step < -(INT_MAX / stride)) error_at( "Global address offset exceeds supported integer range", cur_token_loc()); - offset->var_name = gen_name(); - offset->init_val = step * (op == OP_sub ? -stride : stride); - add_insn(parent, bb, OP_load_constant, offset, NULL, NULL, 0, NULL); + int delta = step * (op == OP_sub ? -stride : stride); result->type = address->type; result->ptr_level = address->ptr_level; result->is_global_address = address->is_global_address; result->is_string_literal = address->is_string_literal; result->is_const_qualified = address->is_const_qualified; + result->is_const_pointer = address->is_const_pointer; + result->pointer_const_mask = address->pointer_const_mask; + result->is_volatile = address->is_volatile; + result->pointer_volatile_mask = address->pointer_volatile_mask; result->pointee_func_signature = address->pointee_func_signature; result->address_stride = address->address_stride; - add_insn(parent, bb, OP_add, result, address, offset, 0, NULL); + if (!wide_global_address_operand(address)) { + unsigned int low = (unsigned int) address->init_val + delta; + unsigned int high = (unsigned int) address->init_val_hi + + (delta < 0 ? ~0U : 0) + + (low < (unsigned int) address->init_val); + store_wide_global_word_result(parent, bb, result, low, high); + } else { + var_t *offset = load_constant(parent, bb, delta, TY_int); + add_insn(parent, bb, OP_add, result, address, offset, 0, NULL); + } return true; } } @@ -468,9 +327,14 @@ bool emit_wide_global_word_arithmetic(block_t *parent, /* An address constant is otherwise an operand only of the operators that * test it. Other arithmetic on one is not a constant expression here. */ + bool raw_pointer = + is_pointer_like_value(left) || (right && is_pointer_like_value(right)); + bool comparison = op_is_comparison(op); if ((wide_global_address_operand(left) || - (right && wide_global_address_operand(right))) && - op != OP_log_and && op != OP_log_or) + (right && wide_global_address_operand(right)) || raw_pointer) && + op != OP_log_and && op != OP_log_or && + !(raw_pointer && comparison && !wide_global_address_operand(left) && + (!right || !wide_global_address_operand(right)))) error_at("Global initializer requires a constant value", cur_token_loc()); @@ -481,7 +345,8 @@ bool emit_wide_global_word_arithmetic(block_t *parent, * unsigned int zero-extends; otherwise each narrow operand extends by its * own signedness. A shift converts only its left operand, by promotion. */ - if (right && op != OP_lshift && op != OP_rshift) { + if (right && op != OP_lshift && op != OP_rshift && + !is_pointer_like_value(left) && !is_pointer_like_value(right)) { type_t *common = integer_common_type(left, right); bool narrow_unsigned = common->size <= TY_int->size && common->is_unsigned; @@ -492,7 +357,8 @@ bool emit_wide_global_word_arithmetic(block_t *parent, if (left->type->size <= TY_int->size) hi = wide_global_narrow_high( lo, left->type->is_unsigned || narrow_unsigned); - } else if (left->type->size <= TY_int->size) { + } else if (!is_pointer_like_value(left) && + left->type->size <= TY_int->size) { hi = wide_global_narrow_high(lo, left->type->is_unsigned); } @@ -502,52 +368,21 @@ bool emit_wide_global_word_arithmetic(block_t *parent, * representation: it must also self-host on targets without a complete * host-level 64-bit value ABI. */ - if (op == OP_eq || op == OP_neq || op == OP_lt || op == OP_leq || - op == OP_gt || op == OP_geq) { - type_t *common = integer_common_type(left, right); - bool equal; - bool less; - bool comparison; - - if (common->size <= TY_int->size) { - equal = lo == rhs_lo; - if (common->is_unsigned) - less = lo < rhs_lo; - else if ((lo ^ rhs_lo) & 0x80000000U) - less = lo & 0x80000000U; - else - less = lo < rhs_lo; - } else { - equal = hi == rhs_hi && lo == rhs_lo; - if (common->is_unsigned) - less = hi < rhs_hi || (hi == rhs_hi && lo < rhs_lo); - else if ((hi ^ rhs_hi) & 0x80000000U) - less = hi & 0x80000000U; - else - less = hi < rhs_hi || (hi == rhs_hi && lo < rhs_lo); - } - - switch (op) { - case OP_eq: - comparison = equal; - break; - case OP_neq: - comparison = !equal; - break; - case OP_lt: - comparison = less; - break; - case OP_leq: - comparison = less || equal; - break; - case OP_gt: - comparison = !less && !equal; - break; - default: - comparison = !less; - break; + if (comparison) { + if (is_pointer_like_value(left) != is_pointer_like_value(right)) { + var_t *integer = is_pointer_like_value(left) ? right : left; + if (!integer->is_const || integer->init_val || integer->init_val_hi) + error_at("Global pointer comparison requires a null constant", + cur_token_loc()); } - store_wide_global_word_result(parent, bb, result, comparison, 0); + bool is_unsigned = is_pointer_like_value(left) || + is_pointer_like_value(right) || + integer_common_type(left, right)->is_unsigned; + pp_integer_t lhs = {.lo = lo, .hi = hi, .is_unsigned = is_unsigned}; + pp_integer_t rhs = { + .lo = rhs_lo, .hi = rhs_hi, .is_unsigned = is_unsigned}; + store_wide_global_word_result(parent, bb, result, + pp_compare(op, &lhs, &rhs), 0); return true; } @@ -564,15 +399,15 @@ bool emit_wide_global_word_arithmetic(block_t *parent, } if (op == OP_div || op == OP_mod) { - unsigned int rem_lo = 0, rem_hi = 0, quo_lo = 0, quo_hi = 0; - /* The usual arithmetic conversions choose the signedness: a signed long * long divided by an unsigned int stays signed long long. */ type_t *common = integer_common_type(left, right); bool is_unsigned = common->is_unsigned; - bool neg_left = !is_unsigned && (hi >> 31); - bool neg_right = !is_unsigned && (rhs_hi >> 31); + pp_integer_t dividend = { + .lo = lo, .hi = hi, .is_unsigned = is_unsigned}; + pp_integer_t divisor = { + .lo = rhs_lo, .hi = rhs_hi, .is_unsigned = is_unsigned}; if (rhs_lo == 0 && rhs_hi == 0) error_at("division by zero in global constant expression", @@ -586,47 +421,9 @@ bool emit_wide_global_word_arithmetic(block_t *parent, (common->size <= TY_int->size ? lo == 0x80000000U : hi == 0x80000000U && lo == 0)) error_at("Overflow in constant expression", cur_token_loc()); - if (neg_left) { - lo = ~lo + 1; - hi = ~hi + (lo == 0); - } - if (neg_right) { - rhs_lo = ~rhs_lo + 1; - rhs_hi = ~rhs_hi + (rhs_lo == 0); - } - for (int i = 0; i < 64; i++) { - unsigned int incoming = hi >> 31; - - hi = (hi << 1) | (lo >> 31); - lo <<= 1; - rem_hi = (rem_hi << 1) | (rem_lo >> 31); - rem_lo = (rem_lo << 1) | incoming; - quo_hi = (quo_hi << 1) | (quo_lo >> 31); - quo_lo <<= 1; - if (rem_hi > rhs_hi || (rem_hi == rhs_hi && rem_lo >= rhs_lo)) { - unsigned int borrow = rem_lo < rhs_lo; - - rem_lo -= rhs_lo; - rem_hi = rem_hi - rhs_hi - borrow; - quo_lo |= 1; - } - } - if (op == OP_div) { - out_lo = quo_lo; - out_hi = quo_hi; - if (neg_left != neg_right) { - out_lo = ~out_lo + 1; - out_hi = ~out_hi + (out_lo == 0); - } - } else { - out_lo = rem_lo; - out_hi = rem_hi; - if (neg_left) { - out_lo = ~out_lo + 1; - out_hi = ~out_hi + (out_lo == 0); - } - } - store_wide_global_word_result(parent, bb, result, out_lo, out_hi); + pp_divmod(÷nd, &divisor, op == OP_mod); + store_wide_global_word_result(parent, bb, result, dividend.lo, + dividend.hi); return true; } @@ -727,8 +524,7 @@ static var_t *promote_wide_global_operand(block_t *parent, if (value->ptr_level || wide_global_address_operand(value) || value->type->size >= TY_int->size) return value; - promoted = require_typed_var(parent, TY_int); - promoted->var_name = gen_name(); + promoted = name_var(require_typed_var(parent, TY_int)); promoted->init_val = value->init_val; promoted->init_val_hi = wide_global_narrow_high(value->init_val, false); promoted->is_const = true; @@ -736,10 +532,37 @@ static var_t *promote_wide_global_operand(block_t *parent, return promoted; } -/* A grouped primary is lowered into the same global setup block as its parent. - * The caller owns the closing parenthesis, so get_operator() naturally stops an - * inner precedence stack without consuming its delimiter. - */ +/* Parse discarded operands for their type without emitting startup effects. */ +static var_t *read_unevaluated_global_value(block_t *parent, + basic_block_t *bb, + block_t *scope, + void (*reader)(block_t *, + basic_block_t **)) +{ + basic_block_t *scratch = bb_create(scope); + int saved_depth = wide_global_unevaluated_depth; + wide_global_unevaluated_depth = 0; + unevaluated_expression_depth++; + reader(scope, &scratch); + unevaluated_expression_depth--; + wide_global_unevaluated_depth = saved_depth; + var_t *operand = opstack_pop(); + var_t *value = name_var( + require_typed_ptr_var(parent, operand->type, operand->ptr_level)); + value->is_func = operand->is_func; + value->func_signature = operand->func_signature; + value->pointee_func_signature = operand->pointee_func_signature; + copy_call_result_array_shape(value, operand); + fixed_array_shape_t shape = fixed_array_shape_from_var(operand); + fixed_array_shape_to_var(value, &shape); + value->has_unsized_array = operand->has_unsized_array; + value->is_const = operand->is_const; + value->init_val = operand->is_const ? operand->init_val : 0; + value->init_val_hi = operand->is_const ? operand->init_val_hi : 0; + add_insn(parent, bb, OP_load_constant, value, NULL, NULL, 0, NULL); + return value; +} + var_t *read_wide_global_literal_primary(block_t *parent, basic_block_t *bb, block_t *scope) @@ -747,6 +570,10 @@ var_t *read_wide_global_literal_primary(block_t *parent, char literal[MAX_TOKEN_LEN]; var_t *value; + if (wide_global_unevaluated_depth) + return read_unevaluated_global_value(parent, bb, scope, + read_expr_operand); + /* Keep wide unary operators out of the legacy word-sized constant * evaluator. Besides making `~0ULL` usable in a static initializer, the * recursive form gives grouped operands and repeated unary operators the @@ -774,8 +601,7 @@ var_t *read_wide_global_literal_primary(block_t *parent, zero->type = value->type; zero->init_val = 0; add_insn(parent, bb, OP_load_constant, zero, NULL, NULL, 0, NULL); - result = require_var(parent); - result->var_name = gen_name(); + result = require_named_var(parent); result->type = value->type; emit_wide_global_word_arithmetic(parent, bb, result, OP_sub, zero, value); @@ -786,8 +612,7 @@ var_t *read_wide_global_literal_primary(block_t *parent, value = promote_wide_global_operand( parent, bb, read_wide_global_literal_primary(parent, bb, scope)); - result = require_var(parent); - result->var_name = gen_name(); + result = require_named_var(parent); result->type = value->type; emit_wide_global_word_arithmetic(parent, bb, result, OP_bit_not, value, NULL); @@ -797,16 +622,19 @@ var_t *read_wide_global_literal_primary(block_t *parent, var_t *result; value = read_wide_global_literal_primary(parent, bb, scope); - result = require_typed_var(parent, TY_int); - result->var_name = gen_name(); + result = name_var(require_typed_var(parent, TY_int)); result->init_val = !wide_global_operand_is_true(value); result->init_val_hi = 0; result->is_const = true; - if (!wide_global_unevaluated_depth) - add_insn(parent, bb, OP_log_not, result, value, NULL, 0, NULL); + add_insn(parent, bb, OP_log_not, result, value, NULL, 0, NULL); return result; } + func_t *cast_signature = read_global_function_pointer_cast(scope); + if (cast_signature) + return read_global_function_cast_value(parent, bb, scope, + cast_signature); + /* An address constant may be an operand here, as the arms of `1 ? "a" : * "b"` are. Every address form belongs to the aggregate constant reader, * which reads this one operand rather than the whole expression it is part @@ -814,19 +642,19 @@ var_t *read_wide_global_literal_primary(block_t *parent, * above. */ if (!subscripted_string_literal_starts_here() && - (lex_peek(T_string, NULL) || + (lex_peek(T_string, NULL) || lex_peek(T_wstring, NULL) || global_address_operand_starts_here(scope))) { basic_block_t *address_bb = bb; var_t *address; - global_tested_operand_depth++; - address = parse_global_constant_value(parent, &address_bb); - global_tested_operand_depth--; + global_constant_context_t ctx = {0}; + ctx.scope = scope; + address = read_global_constant_primary(parent, &address_bb, &ctx); return address; } if (lex_accept(T_sizeof)) { - value = require_typed_var(parent, find_type("size_t", true)); - value->var_name = gen_name(); + value = + name_var(require_typed_var(parent, find_builtin_type("size_t"))); value->init_val = read_sizeof_constant(scope); value->init_val_hi = 0; value->is_const = true; @@ -856,11 +684,11 @@ var_t *read_wide_global_literal_primary(block_t *parent, operand = read_wide_global_literal_primary(parent, bb, scope); hi = (unsigned int) operand->init_val_hi; - if (operand->type->size <= TY_int->size) + if (!is_pointer_like_value(operand) && + operand->type->size <= TY_int->size) hi = wide_global_narrow_high(operand->init_val, operand->type->is_unsigned); - value = require_typed_var(parent, type); - value->var_name = gen_name(); + value = name_var(require_typed_var(parent, type)); fold_integer_constant_cast(value, (unsigned int) operand->init_val, hi); value->is_const = true; add_insn(parent, bb, OP_load_constant, value, NULL, NULL, 0, NULL); @@ -874,12 +702,15 @@ var_t *read_wide_global_literal_primary(block_t *parent, if (lex_peek(T_identifier, literal)) { constant_t *constant = find_scoped_constant(literal, scope); - if (!constant) - error_at("Typed global initializer needs a constant operand", - next_token_loc()); + if (!constant) { + int operand = read_const_expr_operand(scope); + value = load_constant(parent, bb, operand, TY_int); + value->is_const = true; + value->init_val_hi = wide_global_narrow_high(operand, false); + return value; + } lex_expect(T_identifier); - value = require_typed_var(parent, TY_int); - value->var_name = gen_name(); + value = name_var(require_typed_var(parent, TY_int)); /* Every operand this reader returns carries its high word, the * extension of an int-sized value, so no consumer rebuilds it. @@ -900,8 +731,7 @@ var_t *read_wide_global_literal_primary(block_t *parent, error_at("Wide string literal element is not a constant", next_token_loc()); read_string_literal_element(scope, &element); - value = require_typed_var(parent, TY_int); - value->var_name = gen_name(); + value = name_var(require_typed_var(parent, TY_int)); value->init_val = element; value->init_val_hi = wide_global_narrow_high(element, false); value->is_const = true; @@ -934,9 +764,48 @@ var_t *read_wide_global_literal_primary(block_t *parent, * subexpressions, without sending a high word through the legacy int-only * constant evaluator. */ -var_t *read_wide_global_literal_expression(block_t *parent, +static void reduce_wide_global_top_op(block_t *parent, + basic_block_t *bb, + opcode_t *op_stack, + bool *protected_rhs, + var_t **val_stack, + int *op_count, + int *value_count) +{ + var_t *right = val_stack[--*value_count]; + var_t *left = val_stack[--*value_count]; + int top = --*op_count; + var_t *result = require_var(parent); + + if (protected_rhs[top]) + wide_global_unevaluated_depth--; + result->var_name = gen_name(); + result->type = op_stack[top] == OP_log_and || op_stack[top] == OP_log_or + ? TY_int + : integer_binary_result_type(op_stack[top], left, right); + if (!emit_wide_global_word_arithmetic(parent, bb, result, op_stack[top], + left, right)) + add_insn(parent, bb, op_stack[top], result, left, right, 0, NULL); + val_stack[(*value_count)++] = result; +} + +static void reduce_wide_global_pending_ops(block_t *parent, basic_block_t *bb, - block_t *scope) + opcode_t *op_stack, + bool *protected_rhs, + var_t **val_stack, + int *op_count, + int *value_count) +{ + while (*op_count > 0) + reduce_wide_global_top_op(parent, bb, op_stack, protected_rhs, + val_stack, op_count, value_count); +} + +var_t *read_wide_global_literal_tail(block_t *parent, + basic_block_t *bb, + block_t *scope, + var_t *first) { opcode_t op_stack[MAX_OPERATOR_STACK_SIZE]; bool protected_rhs[MAX_OPERATOR_STACK_SIZE] = {0}; @@ -944,8 +813,8 @@ var_t *read_wide_global_literal_expression(block_t *parent, int op_stack_index = 0, val_stack_index = 0; opcode_t op; - val_stack[val_stack_index++] = promote_wide_global_operand( - parent, bb, read_wide_global_literal_primary(parent, bb, scope)); + val_stack[val_stack_index++] = + promote_wide_global_operand(parent, bb, first); op = get_operator(); while (op != OP_generic) { if (op == OP_ternary) { @@ -959,24 +828,9 @@ var_t *read_wide_global_literal_expression(block_t *parent, * operator first so the condition is the complete expression, not * merely the primary immediately before `?`. */ - while (op_stack_index > 0) { - var_t *right = val_stack[--val_stack_index]; - var_t *left = val_stack[--val_stack_index]; - var_t *result = require_var(parent); - - op_stack_index--; - if (protected_rhs[op_stack_index]) - wide_global_unevaluated_depth--; - result->var_name = gen_name(); - result->type = integer_binary_result_type( - op_stack[op_stack_index], left, right); - if (!emit_wide_global_word_arithmetic(parent, bb, result, - op_stack[op_stack_index], - left, right)) - add_insn(parent, bb, op_stack[op_stack_index], result, left, - right, 0, NULL); - val_stack[val_stack_index++] = result; - } + reduce_wide_global_pending_ops(parent, bb, op_stack, protected_rhs, + val_stack, &op_stack_index, + &val_stack_index); condition = val_stack[--val_stack_index]; condition_true = wide_global_operand_is_true(condition); @@ -987,23 +841,34 @@ var_t *read_wide_global_literal_expression(block_t *parent, * through the legacy int-only evaluator. */ lex_expect(T_question); - if (!condition_true) - wide_global_unevaluated_depth++; - when_true = read_wide_global_literal_expression(parent, bb, scope); - if (!condition_true) - wide_global_unevaluated_depth--; + when_true = + !condition_true + ? read_unevaluated_global_value(parent, bb, scope, + read_control_expression) + : read_wide_global_literal_expression(parent, bb, scope); lex_expect(T_colon); - if (condition_true) - wide_global_unevaluated_depth++; - when_false = read_wide_global_literal_expression(parent, bb, scope); - if (condition_true) - wide_global_unevaluated_depth--; + when_false = + condition_true + ? read_unevaluated_global_value( + parent, bb, scope, read_assignment_or_expression) + : read_wide_global_literal_expression(parent, bb, scope); /* An address constant has no arithmetic conversion to make with the * other arm; the selected one is the value. */ - if (wide_global_address_operand(when_true) || - wide_global_address_operand(when_false)) + bool true_pointer = wide_global_address_operand(when_true) || + is_pointer_like_value(when_true) || + when_true->has_unsized_array; + bool false_pointer = wide_global_address_operand(when_false) || + is_pointer_like_value(when_false) || + when_false->has_unsized_array; + if (true_pointer != false_pointer && + !is_null_pointer_constant(true_pointer ? when_false + : when_true)) + error_at( + "Conditional pointer operands must be pointers or null", + cur_token_loc()); + if (true_pointer || false_pointer) return condition_true ? when_true : when_false; common = integer_common_type(when_true, when_false); normalize_integer_binary_operands(parent, &bb, OP_add, &when_true, @@ -1016,23 +881,10 @@ var_t *read_wide_global_literal_expression(block_t *parent, } while (op_stack_index > 0 && get_operator_prio(op_stack[op_stack_index - 1]) >= - get_operator_prio(op)) { - var_t *right = val_stack[--val_stack_index]; - var_t *left = val_stack[--val_stack_index]; - var_t *result = require_var(parent); - - op_stack_index--; - if (protected_rhs[op_stack_index]) - wide_global_unevaluated_depth--; - result->var_name = gen_name(); - result->type = integer_binary_result_type(op_stack[op_stack_index], - left, right); - if (!emit_wide_global_word_arithmetic( - parent, bb, result, op_stack[op_stack_index], left, right)) - add_insn(parent, bb, op_stack[op_stack_index], result, left, - right, 0, NULL); - val_stack[val_stack_index++] = result; - } + get_operator_prio(op)) + reduce_wide_global_top_op(parent, bb, op_stack, protected_rhs, + val_stack, &op_stack_index, + &val_stack_index); if (op_stack_index >= MAX_OPERATOR_STACK_SIZE || val_stack_index >= MAX_OPERATOR_STACK_SIZE) fatal("Wide global initializer is too complex"); @@ -1053,92 +905,43 @@ var_t *read_wide_global_literal_expression(block_t *parent, parent, bb, read_wide_global_literal_primary(parent, bb, scope)); op = get_operator(); } - while (op_stack_index > 0) { - var_t *right = val_stack[--val_stack_index]; - var_t *left = val_stack[--val_stack_index]; - var_t *result = require_var(parent); - - op_stack_index--; - if (protected_rhs[op_stack_index]) - wide_global_unevaluated_depth--; - result->var_name = gen_name(); - result->type = - integer_binary_result_type(op_stack[op_stack_index], left, right); - if (!emit_wide_global_word_arithmetic( - parent, bb, result, op_stack[op_stack_index], left, right)) - add_insn(parent, bb, op_stack[op_stack_index], result, left, right, - 0, NULL); - val_stack[val_stack_index++] = result; - } + reduce_wide_global_pending_ops(parent, bb, op_stack, protected_rhs, + val_stack, &op_stack_index, + &val_stack_index); return val_stack[0]; } -/* Skip the arm of a conditional expression that its constant condition - * discards. The arm may itself hold grouped or nested conditionals, so count - * the brackets and the '?' still waiting for their ':' rather than stopping at - * the first ':'. A second operand ends at its matching ':'; a third ends at the - * ',' or ';' ending the declarator, the ':' of an enclosing conditional or the - * bracket closing an enclosing initializer. - */ -static void skip_discarded_conditional_arm(bool second_operand) +var_t *read_wide_global_literal_expression(block_t *parent, + basic_block_t *bb, + block_t *scope) { - int depth = 0; - int pending = 0; - - for (;;) { - token_t *next = cur_token->next; - - if (!next || next->kind == T_eof) - return; - if (next->kind == T_open_bracket || next->kind == T_open_square || - next->kind == T_open_curly) { - depth++; - } else if (next->kind == T_close_bracket || - next->kind == T_close_square || - next->kind == T_close_curly) { - if (depth == 0) - return; - depth--; - } else if (depth == 0 && next->kind == T_question) { - pending++; - } else if (depth == 0 && next->kind == T_colon) { - if (pending == 0) - return; - pending--; - } else if (depth == 0 && !second_operand && - (next->kind == T_comma || next->kind == T_semicolon)) { - return; - } - lex_next(); - } + return read_wide_global_literal_tail( + parent, bb, scope, read_wide_global_literal_primary(parent, bb, scope)); } -void eval_ternary_imm(int cond, var_t *var) +static void validate_global_compound_address(const var_t *dest, + const var_t *literal) { - if (cond == 0) { - skip_discarded_conditional_arm(true); - lex_expect(T_colon); - read_global_assignment_var(var); - } else { - read_global_assignment_var(var); - lex_expect(T_colon); - skip_discarded_conditional_arm(false); + if (literal->is_func) { + func_t *slot = dest->pointee_func_signature; + if (!slot && dest->ptr_level == 1 && dest->type->func_signature && + !dest->type->is_direct_function_type) + slot = dest->type->func_signature; + if (!slot || dest->array_size || + !compatible_function_signature(slot, literal->func_signature)) + error_at("Incompatible compound literal address", cur_token_loc()); + } else if (dest->array_size || dest->ptr_level != literal->ptr_level + 1 || + !(compatible_decl_type(dest->type, literal->type) || + (dest->type == TY_void && !literal->ptr_level))) { + error_at("Incompatible compound literal address", cur_token_loc()); } } bool read_global_assignment_var(var_t *var) { - var_t *vd, *rs1; - - /* A block-scope static is lowered in the global setup block, but its - * initializer is parsed in the declaration's lexical scope. In particular - * an enumerator declared by an enclosing block remains an integer constant - * expression here. - */ block_t *scope = var->scope ? var->scope : GLOBAL_BLOCK; block_t *parent = GLOBAL_BLOCK; basic_block_t *bb = GLOBAL_FUNC->bbs; - validate_string_array_initializer(var); if ((var->array_size > 0 || var->has_unsized_array) && is_char_array(var) && lex_peek(T_string, NULL)) { @@ -1150,462 +953,15 @@ bool read_global_assignment_var(var_t *var) parse_wstring_array_init(var, parent, &bb); return true; } - - /* An address constant that an operator only tests, as in `"a" && 1`, is the - * operand of a constant expression rather than this object's value. - */ - if (global_tested_operand_starts_here(scope)) { - var_t *tested = read_wide_global_literal_expression(parent, bb, scope); - - reject_global_integer_pointer(var, tested); - add_insn(parent, bb, OP_assign, var, tested, NULL, 0, NULL); - return true; - } - - /* A cast to a function pointer type converts the function designator or - * null pointer constant it applies to. - */ - { - func_t *cast_signature = read_global_function_pointer_cast(scope); - - if (cast_signature) { - func_t *saved_signature = global_function_cast_signature; - - global_function_cast_signature = cast_signature; - read_global_assignment_var(var); - global_function_cast_signature = saved_signature; - return true; - } - } - - /* global initialization must be constant */ - if (global_pointer_cast_starts_here(scope)) { - int saved_stride = global_pointer_cast_stride; - func_t *slot_signature; - int stride = read_global_pointer_cast(scope, &slot_signature); - - /* In a chain of casts the outermost one decides the stride. */ - if (saved_stride) - stride = saved_stride; - if (!global_address_operand_starts_here(scope)) { - rs1 = read_global_cast_integer_address(parent, bb, scope, stride, - slot_signature); - if (rs1->pointee_func_signature) - diagnose_callback_slot_initializer(rs1, var); - emit_global_scalar_assignment(parent, bb, var, rs1); - return true; - } - global_pointer_cast_stride = stride; - read_global_assignment_var(var); - global_pointer_cast_stride = saved_stride; - return true; - } - { - /* A function designator is a valid address constant. Keep it as the - * function symbol until lowering: OP_address_of_func has the deferred - * relocation needed because the target function's code offset is not - * known while global initializers are parsed. - */ - bool address_dereference = - global_function_address_dereference_starts_here(); - token_t *address_dereference_identifier = NULL; - bool explicit_address; - bool grouped_function_designator = false; - - if (address_dereference) { - var_t *addr; - var_t *symbol; - - address_dereference_identifier = - consume_global_function_address_dereference(); - if (!find_visible_func(address_dereference_identifier->literal, - scope)) - error_at("Function address requires a visible declaration", - cur_token_loc()); - if (!var->is_func && !var->ptr_level && - !(var->type && var->type->ptr_level)) - error_at("Function address requires a pointer initializer", - cur_token_loc()); - addr = require_ref_var(parent, var->type, var->ptr_level); - symbol = require_func_symbol_var(parent); - addr->var_name = gen_name(); - symbol->is_func = true; - symbol->var_name = - intern_string(address_dereference_identifier->literal); - diagnose_global_function_conversion(symbol, var); - add_insn(parent, bb, OP_address_of, addr, var, NULL, 0, NULL); - add_insn(parent, bb, OP_write, NULL, addr, symbol, PTR_SIZE, NULL); - return true; - } - explicit_address = lex_accept(T_ampersand); - if (grouped_global_function_designator_starts_here(false)) { - lex_expect(T_open_bracket); - grouped_function_designator = true; - } - char token[MAX_ID_LEN]; - if (lex_peek(T_identifier, token)) { - func_t *func = find_visible_func(token, scope); - if (func) { - if (!var->is_func && !var->ptr_level && - !(var->type && var->type->ptr_level)) - error_at("Function address requires a pointer initializer", - cur_token_loc()); - var_t *addr = - require_ref_var(parent, var->type, var->ptr_level); - var_t *symbol = require_func_symbol_var(parent); - - addr->var_name = gen_name(); - symbol->is_func = true; - symbol->var_name = intern_string(token); - diagnose_global_function_conversion(symbol, var); - lex_expect(T_identifier); - if (grouped_function_designator) - lex_expect(T_close_bracket); - add_insn(parent, bb, OP_address_of, addr, var, NULL, 0, NULL); - add_insn(parent, bb, OP_write, NULL, addr, symbol, PTR_SIZE, - NULL); - return true; - } - - /* Static locals have global storage but lexical visibility. Use the - * declaration scope for name resolution while continuing to emit - * their initializer into the synthetic global block. - */ - var_t *object = find_var(token, scope); - if (object && object->is_global && - (explicit_address || object->array_size)) { - fixed_array_shape_t shape = fixed_array_shape_from_var(object); - var_t *object_addr = - require_ref_var(parent, object->type, object->ptr_level); - - object_addr->var_name = gen_name(); - object_addr->is_global_address = true; - - /* Taking the address of a callback object creates a slot. - * Retain the callback prototype on that non-callable outer - * pointer so the global initializer conversion below has the - * same information as block-scope `&callback`. - */ - object_addr->pointee_func_signature = - object->pointee_func_signature - ? object->pointee_func_signature - : object->func_signature; - lex_expect(T_identifier); - add_insn(parent, bb, OP_address_of, object_addr, object, NULL, - 0, NULL); - if (!explicit_address && object->array_dim2 && - lex_peek(T_open_square, NULL)) { - object_addr = read_global_address_designator( - scope, parent, &bb, &object, object_addr, true); - } else if (!explicit_address && object->array_size && - (lex_peek(T_plus, NULL) || - lex_peek(T_minus, NULL))) { - int index = read_global_address_offset(scope, parent, bb); - int stride = object->ptr_level || object->is_func - ? PTR_SIZE - : object->type->size; - var_t *byte_offset = require_var(parent); - var_t *offset_addr = require_ref_var(parent, object->type, - object->ptr_level); - - stride = global_pointer_cast_stride - ? global_pointer_cast_stride - : fixed_array_shape_stride(&shape, 0, stride); - byte_offset->var_name = gen_name(); - byte_offset->init_val = index * stride; - add_insn(parent, bb, OP_load_constant, byte_offset, NULL, - NULL, 0, NULL); - offset_addr->var_name = gen_name(); - offset_addr->is_global_address = true; - offset_addr->pointee_func_signature = - object_addr->pointee_func_signature; - add_insn(parent, bb, OP_add, offset_addr, object_addr, - byte_offset, 0, NULL); - object_addr = offset_addr; - } - if (explicit_address) - object_addr = read_global_address_designator( - scope, parent, &bb, &object, object_addr, false); - diagnose_callback_slot_initializer(object_addr, var); - add_insn(parent, bb, OP_assign, var, object_addr, NULL, 0, - NULL); - return true; - } - } - if (explicit_address && subscripted_string_literal_starts_here()) { - rs1 = read_string_literal_element_address(parent, bb, scope); - diagnose_const_pointer_conversion(rs1, var); - emit_global_scalar_assignment(parent, bb, var, rs1); - return true; - } - if (explicit_address && scope == GLOBAL_BLOCK && - global_compound_literal_starts_here()) { - /* A compound literal at file scope has static storage (C99 - * 6.5.2.5p6), so `&(int){8}` is an address constant. Give the - * literal an unnamed global and initialize the pointer with its - * address. - */ - type_t *literal_type; - var_t *literal; - var_t *literal_addr; - int literal_ptr_level = 0; - func_t *callback = NULL; - - lex_expect(T_open_bracket); - literal_type = read_type_name_specifiers(GLOBAL_BLOCK); - while (lex_accept(T_asterisk)) - literal_ptr_level++; - - /* `&(int (*)(int)){f}` and `&(callback_t){f}` address a callback - * object; the pointer they initialize is a callback slot. - */ - if (literal_type && abstract_function_pointer_follows()) { - int callback_level; - - callback = read_abstract_function_pointer( - literal_type, literal_ptr_level, &callback_level); - if (callback_level != 1) - error_at("Incompatible compound literal address", - cur_token_loc()); - } else if (literal_type && literal_type->func_signature && - !literal_type->is_direct_function_type && - !literal_type->array_size && !literal_ptr_level) { - callback = literal_type->func_signature; - } - lex_expect(T_close_bracket); - if (callback) { - func_t *slot = var->pointee_func_signature; - - if (!slot && var->ptr_level == 1 && var->type->func_signature && - !var->type->is_direct_function_type) - slot = var->type->func_signature; - if (!slot || var->array_size || - !compatible_function_signature(slot, callback)) - error_at("Incompatible compound literal address", - cur_token_loc()); - literal = require_typed_var(GLOBAL_BLOCK, literal_type); - literal->var_name = gen_name(); - literal->is_global = true; - literal->ptr_level = - literal_type->func_signature ? 0 : literal_ptr_level; - literal->is_func = true; - literal->func_signature = callback; - add_insn(GLOBAL_BLOCK, bb, OP_allocat, literal, NULL, NULL, 0, - NULL); - lex_expect(T_open_curly); - if (lex_peek(T_close_curly, NULL)) - error_at("Scalar compound literal needs an initializer", - next_token_loc()); - read_global_assignment_var(literal); - lex_accept(T_comma); - lex_expect(T_close_curly); - literal_addr = - require_ref_var(parent, literal->type, literal->ptr_level); - literal_addr->var_name = gen_name(); - literal_addr->is_global_address = true; - literal_addr->pointee_func_signature = callback; - add_insn(parent, bb, OP_address_of, literal_addr, literal, NULL, - 0, NULL); - add_insn(parent, bb, OP_assign, var, literal_addr, NULL, 0, - NULL); - return true; - } - if (!literal_type || literal_type->array_size || - literal_type->func_signature || var->array_size || - var->ptr_level != literal_ptr_level + 1 || - !(compatible_decl_type(var->type, literal_type) || - (var->type == TY_void && !literal_ptr_level))) - error_at("Incompatible compound literal address", - cur_token_loc()); - literal = require_typed_var(GLOBAL_BLOCK, literal_type); - literal->var_name = gen_name(); - literal->is_global = true; - literal->ptr_level = literal_ptr_level; - add_insn(GLOBAL_BLOCK, bb, OP_allocat, literal, NULL, NULL, 0, - NULL); - if (!literal_ptr_level && is_record_type(literal_type)) { - parse_global_record_init(literal, GLOBAL_BLOCK); - } else { - lex_expect(T_open_curly); - if (lex_peek(T_close_curly, NULL)) - error_at("Scalar compound literal needs an initializer", - next_token_loc()); - read_global_assignment_var(literal); - lex_accept(T_comma); - lex_expect(T_close_curly); - } - literal_addr = - require_ref_var(parent, literal->type, literal->ptr_level); - literal_addr->var_name = gen_name(); - literal_addr->is_global_address = true; - add_insn(parent, bb, OP_address_of, literal_addr, literal, NULL, 0, - NULL); - add_insn(parent, bb, OP_assign, var, literal_addr, NULL, 0, NULL); - return true; - } - if (explicit_address) - error_at("Expected a global object or function after '&'", - cur_token_loc()); - if (string_address_offset_starts_here()) { - rs1 = read_string_address_offset(parent, bb, scope); - diagnose_const_pointer_conversion(rs1, var); - emit_global_scalar_assignment(parent, bb, var, rs1); - return true; - } - - /* The legacy global evaluator stores operands in int. Parse a wide - * literal-only expression separately so its upper payload survives; - * lower each reduction into the global setup block instead of trying to - * narrow the expression through that evaluator. - */ - if (initializer_needs_wide_reader(cur_token->next, scope, true) || - string_element_appears_before_initializer_end(cur_token->next)) { - rs1 = read_wide_global_literal_expression(parent, bb, scope); - reject_global_integer_pointer(var, rs1); - add_insn(parent, bb, OP_assign, var, rs1, NULL, 0, NULL); - return true; - } - if ((lex_peek(T_string, NULL) || lex_peek(T_wstring, NULL)) && - !subscripted_string_literal_starts_here()) { - /* String literal global initialization: String literals are now - * stored in .rodata section. TODO: Implement compile-time address - * resolution for global pointer initialization with rodata - * addresses (e.g., char *p = "str";) - */ - if (lex_peek(T_wstring, NULL)) - read_wstring_param(parent, bb); - else - read_literal_param(parent, bb); - rs1 = opstack_pop(); - vd = var; - diagnose_const_pointer_conversion(rs1, vd); - emit_global_scalar_assignment(parent, bb, vd, rs1); - return true; - } - - opcode_t op_stack[MAX_OPERATOR_STACK_SIZE]; - opcode_t op, next_op; - int val_stack[MAX_OPERATOR_STACK_SIZE]; - int op_stack_index = 0, val_stack_index = 0; - int operand1, operand2; - operand1 = read_const_expr_operand(scope); - op = get_operator(); - /* only one value after assignment */ - if (op == OP_generic) { - vd = require_var(parent); - vd->var_name = gen_name(); - vd->init_val = operand1; - add_insn(parent, bb, OP_load_constant, vd, NULL, NULL, 0, NULL); - - rs1 = vd; - emit_global_scalar_assignment(parent, bb, var, rs1); - return true; - } - if (op == OP_ternary) { - lex_expect(T_question); - eval_ternary_imm(operand1, var); - return true; - } - operand2 = read_const_expr_operand(scope); - next_op = get_operator(); - if (next_op == OP_generic) { - /* only two operands, apply and return */ - vd = require_var(parent); - vd->var_name = gen_name(); - vd->init_val = eval_expression_imm(op, operand1, operand2); - add_insn(parent, bb, OP_load_constant, vd, NULL, NULL, 0, NULL); - - rs1 = vd; - reject_global_integer_pointer(var, rs1); - add_insn(parent, bb, OP_assign, var, rs1, NULL, 0, NULL); - return true; - } - - /* using stack if operands more than two */ - op_stack[op_stack_index++] = op; - op = next_op; - val_stack[val_stack_index++] = operand1; - val_stack[val_stack_index++] = operand2; - - while (op != OP_generic && op != OP_ternary) { - if (op_stack_index > 0) { - /* we have a continuation, use stack */ - int same_op = 0; - do { - opcode_t stack_op = op_stack[op_stack_index - 1]; - if (get_operator_prio(stack_op) >= get_operator_prio(op)) { - operand1 = val_stack[val_stack_index - 2]; - operand2 = val_stack[val_stack_index - 1]; - val_stack_index -= 2; - - /* apply stack operator and push result back */ - val_stack[val_stack_index++] = - eval_expression_imm(stack_op, operand1, operand2); - - /* pop op stack */ - op_stack_index--; - } else { - same_op = 1; - } - /* continue util next operation is higher prio */ - } while (op_stack_index > 0 && same_op == 0); - } - /* push next operand on stack */ - if (val_stack_index >= MAX_OPERATOR_STACK_SIZE || - op_stack_index >= MAX_OPERATOR_STACK_SIZE) - fatal("Constant expression too complex"); - val_stack[val_stack_index++] = read_const_expr_operand(scope); - /* push operator on stack */ - op_stack[op_stack_index++] = op; - op = get_operator(); - } - /* unwind stack and apply operations */ - while (op_stack_index > 0) { - opcode_t stack_op = op_stack[op_stack_index - 1]; - - /* pop stack and apply operators */ - operand1 = val_stack[val_stack_index - 2]; - operand2 = val_stack[val_stack_index - 1]; - val_stack_index -= 2; - - /* apply stack operator and push value back on stack */ - val_stack[val_stack_index++] = - eval_expression_imm(stack_op, operand1, operand2); - - if (op_stack_index == 1) { - if (op == OP_ternary) { - lex_expect(T_question); - eval_ternary_imm(val_stack[0], var); - } else { - vd = require_var(parent); - vd->var_name = gen_name(); - vd->init_val = val_stack[0]; - add_insn(parent, bb, OP_load_constant, vd, NULL, NULL, 0, - NULL); - - rs1 = vd; - emit_global_scalar_assignment(parent, bb, var, rs1); - } - return true; - } - - /* pop op stack */ - op_stack_index--; - } - if (op == OP_ternary) { - lex_expect(T_question); - eval_ternary_imm(val_stack[0], var); - } else { - vd = require_var(parent); - vd->var_name = gen_name(); - vd->init_val = val_stack[0]; - add_insn(parent, GLOBAL_FUNC->bbs, OP_load_constant, vd, NULL, NULL, - 0, NULL); - - rs1 = vd; - emit_global_scalar_assignment(parent, GLOBAL_FUNC->bbs, var, rs1); - } - return true; - } - return false; + global_constant_context_t ctx = {0}; + ctx.scope = scope; + ctx.dest = var; + var_t *value = read_global_constant_value(parent, &bb, &ctx); + if (!value->is_func) + diagnose_function_pointer_conversion(value, var); + diagnose_callback_slot_initializer(value, var); + if (value->is_string_literal) + diagnose_const_pointer_conversion(value, var); + emit_global_scalar_assignment(parent, bb, var, value); + return true; } diff --git a/src/parser-decl.c b/src/parser-decl.c index 268d9663..ac0c7b95 100644 --- a/src/parser-decl.c +++ b/src/parser-decl.c @@ -85,9 +85,7 @@ sizeof_derivation_t read_sizeof_abstract_declarator(block_t *scope, while (lex_accept(T_asterisk)) { derivation = SIZEOF_DERIVED_POINTER; - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; + skip_type_qualifiers(); } /* A parenthesis opens an inner declarator only where one can begin; @@ -277,7 +275,9 @@ type_t *read_scalar_type_specifiers(bool *is_const, /* Consume the specifier list of a type name used only for its metadata, and * return the type it names, or NULL when none of it names a type. */ -type_t *read_type_name_specifiers(block_t *scope) +static type_t *read_qualified_type_name_specifiers(block_t *scope, + bool *const_out, + bool *volatile_out) { char token[MAX_ID_LEN]; bool is_const = false; @@ -285,8 +285,11 @@ type_t *read_type_name_specifiers(block_t *scope) type_t *type = read_scalar_type_specifiers(&is_const, &is_volatile, NULL); base_type_t record_kind; - if (type) + if (type) { + *const_out |= is_const; + *volatile_out |= is_volatile; return type; + } record_kind = accept_record_keyword(); if (record_kind) { lex_ident(T_identifier, token); @@ -299,13 +302,86 @@ type_t *read_type_name_specifiers(block_t *scope) if (type) lex_expect(T_identifier); } - /* A qualifier may follow the record, enum or typedef name as well. */ - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; + + /* A qualifier may follow the record, enum or typedef name as well; restrict + * qualifies only a pointer typedef. + */ + while (lex_peek(T_const, NULL) || lex_peek(T_volatile, NULL) || + lex_peek(T_restrict, NULL)) { + if (lex_accept(T_const)) + is_const = true; + else if (lex_accept(T_volatile)) + is_volatile = true; + else { + lex_expect(T_restrict); + if (!type || !type->ptr_level) + error_at("restrict requires a pointer type", cur_token_loc()); + } + } + *const_out |= is_const; + *volatile_out |= is_volatile; return type; } +type_t *read_type_name_specifiers(block_t *scope) +{ + bool is_const = false, is_volatile = false; + return read_qualified_type_name_specifiers(scope, &is_const, &is_volatile); +} + +/* Retain qualifiers on the named return type without changing its typedef. */ +static type_t *qualify_type_name(type_t *type, bool is_const, bool is_volatile) +{ + if (!type || !type->ptr_level || type->func_signature || + type->pointee_func_signature || (!is_const && !is_volatile)) + return type; + type_t *qualified = arena_alloc(GENERAL_ARENA, sizeof(*qualified)); + memcpy(qualified, type, sizeof(*qualified)); + if (type->ptr_level <= 32) { + if (is_const) + qualified->pointer_const_mask |= 1U << (type->ptr_level - 1); + if (is_volatile) + qualified->pointer_volatile_mask |= 1U << (type->ptr_level - 1); + } + return qualified; +} + +void init_type_name_decl(var_t *decl, + type_t *type, + bool is_const, + bool is_volatile) +{ + memset(decl, 0, sizeof(*decl)); + decl->type = qualify_type_name(type, is_const, is_volatile); + if (!type || !type->ptr_level) { + decl->is_const_qualified = is_const; + decl->is_volatile = is_volatile; + } +} + +void read_type_name_decl(block_t *scope, var_t *decl) +{ + bool is_const = false, is_volatile = false; + type_t *type = + read_qualified_type_name_specifiers(scope, &is_const, &is_volatile); + init_type_name_decl(decl, type, is_const, is_volatile); + while (lex_accept(T_asterisk)) { + decl->ptr_level++; + while (lex_peek(T_const, NULL) || lex_peek(T_volatile, NULL) || + lex_peek(T_restrict, NULL)) { + if (lex_accept(T_const)) { + if (decl->ptr_level <= 32) + decl->pointer_const_mask |= 1U << (decl->ptr_level - 1); + } else if (lex_accept(T_volatile)) { + int depth = decl->ptr_level + (type ? type->ptr_level : 0); + if (depth <= 32) + decl->pointer_volatile_mask |= 1U << (depth - 1); + } else + lex_expect(T_restrict); + } + } +} + /* Integer constant expressions may contain sizeof(type-name). This parser only * needs type metadata, so keep it separate from expression lowering and avoid * emitting the otherwise unevaluated sizeof IR into an array bound or @@ -364,6 +440,44 @@ static int read_offsetof_array_subscripts(var_t *field, block_t *scope) } } +static int read_offsetof_designator(block_t *scope) +{ + char name[MAX_ID_LEN]; + type_t *record; + var_t *field; + int offset = 0; + bool has_nested_member; + + lex_expect(T_identifier); + lex_expect(T_open_bracket); + base_type_t record_kind = accept_record_keyword(); + if (record_kind) { + lex_ident(T_identifier, name); + record = find_record_tag(name, scope, record_kind); + } else { + lex_ident(T_identifier, name); + record = find_visible_type(name, scope); + } + if (!is_record_type(record)) + error_at("offsetof requires a struct or union type", cur_token_loc()); + lex_expect(T_comma); + do { + lex_ident(T_identifier, name); + field = find_member(name, record); + if (!field) + error_at("Unknown record member", cur_token_loc()); + offset += field->offset; + offset += read_offsetof_array_subscripts(field, scope); + record = field->type; + has_nested_member = lex_accept(T_dot); + if (has_nested_member && (field->ptr_level || !is_record_type(record))) + error_at("Nested offsetof member requires a record", + cur_token_loc()); + } while (has_nested_member); + lex_expect(T_close_bracket); + return offset; +} + /* Whether the next operand is a string literal, with any adjacent ones joined * to it, followed by a subscript, as in `"ab"[1]`. */ @@ -446,45 +560,8 @@ int read_const_expr_operand(block_t *scope) : parse_character_constant(buffer); } if (lex_peek(T_identifier, buffer)) { - if (!strcmp(buffer, "__builtin_offsetof")) { - char type_name[MAX_ID_LEN]; - char member_name[MAX_ID_LEN]; - type_t *record; - var_t *field; - int offset = 0; - bool has_nested_member; - - lex_expect(T_identifier); - lex_expect(T_open_bracket); - base_type_t record_kind = accept_record_keyword(); - if (record_kind) { - lex_ident(T_identifier, type_name); - record = find_record_tag(type_name, scope, record_kind); - } else { - lex_ident(T_identifier, type_name); - record = find_type(type_name, true); - } - if (!is_record_type(record)) - error_at("offsetof requires a struct or union type", - cur_token_loc()); - lex_expect(T_comma); - do { - lex_ident(T_identifier, member_name); - field = find_member(member_name, record); - if (!field) - error_at("Unknown record member", cur_token_loc()); - offset += field->offset; - offset += read_offsetof_array_subscripts(field, scope); - record = field->type; - has_nested_member = lex_accept(T_dot); - if (has_nested_member && - (field->ptr_level || !is_record_type(record))) - error_at("Nested offsetof member requires a record", - cur_token_loc()); - } while (has_nested_member); - lex_expect(T_close_bracket); - return offset; - } + if (!strcmp(buffer, "__builtin_offsetof")) + return read_offsetof_designator(scope); lex_expect(T_identifier); constant_t *con = find_scoped_constant(buffer, scope); if (con) @@ -595,31 +672,41 @@ static bool grouped_declarator_name_follows(block_t *scope, bool is_param) !(is_param && find_visible_type(name->literal, scope)); } +static token_t *function_pointer_declarator_suffix(token_t *open, + int *star_count) +{ + token_t *token = open && open->kind == T_open_bracket ? open->next : NULL; + + if (star_count) + *star_count = 0; + if (!token || token->kind != T_asterisk) + return NULL; + while (token && (token->kind == T_asterisk || token->kind == T_const || + token->kind == T_volatile || token->kind == T_restrict)) { + if (star_count && token->kind == T_asterisk) + (*star_count)++; + token = token->next; + } + return token; +} + /* Whether the abstract declarator of a function pointer type name, such as * `(*)(int)` in `(int (*)(int))` or `(**)(void)`, starts at the next token. */ bool abstract_function_pointer_follows(void) { - token_t *token = cur_token->next; + token_t *token = function_pointer_declarator_suffix(cur_token->next, NULL); - if (!token || token->kind != T_open_bracket || !(token = token->next) || - token->kind != T_asterisk) - return false; - while (token && (token->kind == T_asterisk || token->kind == T_const || - token->kind == T_volatile || token->kind == T_restrict)) - token = token->next; return token && token->kind == T_close_bracket && token->next && token->next->kind == T_open_bracket; } -/* Read that abstract declarator for a type name whose specifiers and leading - * stars named @type and @ptr_level, the return type. +/* Read an abstract callback declarator retaining its complete return type. * * Return the prototype and set @pointer_level to the stars inside the * parentheses. */ -func_t *read_abstract_function_pointer(type_t *type, - int ptr_level, +func_t *read_abstract_function_pointer(const var_t *return_decl, int *pointer_level) { func_t *func = arena_alloc_func(); @@ -628,15 +715,12 @@ func_t *read_abstract_function_pointer(type_t *type, lex_expect(T_open_bracket); while (lex_accept(T_asterisk)) { (*pointer_level)++; - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; + skip_type_qualifiers(); } lex_expect(T_close_bracket); - func->return_def.type = type; - func->return_def.ptr_level = ptr_level; - func->returns_aggregate = - is_record_type(type) && !ptr_level && !type->ptr_level; + memcpy(&func->return_def, return_decl, sizeof(func->return_def)); + func->returns_aggregate = is_record_type(return_decl->type) && + !has_effective_pointer(return_decl); read_parameter_list_decl(func, true); return func; } @@ -661,6 +745,16 @@ static bool abstract_function_parameter_follows(block_t *scope) first->kind == T_volatile; } +/* Array suffixes on a callback object do not belong to its return type. */ +static void set_function_return(var_t *result, const var_t *declarator) +{ + memcpy(result, declarator, sizeof(*result)); + result->array_size = 0; + result->array_dim2 = result->array_dim3 = result->array_dim4 = 0; + result->has_direct_array_declarator = false; + result->has_unsized_array = false; +} + /* Read the parameter list of a parameter declared with a function type, which * C99 6.7.5.3p8 adjusts to a pointer to that function, into @vd with the * representation of `int (*fn)(int)`. @@ -669,7 +763,7 @@ static void read_adjusted_function_parameter(var_t *vd) { func_t *func = arena_alloc_func(); - memcpy(&func->return_def, vd, sizeof(var_t)); + set_function_return(&func->return_def, vd); func->returns_aggregate = is_record_type(func->return_def.type) && !has_effective_pointer(&func->return_def); read_parameter_list_decl(func, true); @@ -678,25 +772,63 @@ static void read_adjusted_function_parameter(var_t *vd) vd->parenthesized_function_pointer_level = 1; } -void read_inner_var_decl(var_t *vd, - bool anon, - bool is_param, - bool is_record_member); +static void read_array_parameter_qualifiers(var_t *vd, + bool is_param, + int dim, + bool *static_bound, + bool *const_bound) +{ + while (lex_peek(T_static, NULL) || lex_peek(T_const, NULL) || + lex_peek(T_volatile, NULL) || lex_peek(T_restrict, NULL)) { + if (!is_param || dim) + error_at("array bracket qualifiers require a parameter", + cur_token_loc()); + if (lex_accept(T_static)) { + if (*static_bound) + error_at("duplicate static array parameter qualifier", + cur_token_loc()); + *static_bound = true; + } else if (lex_accept(T_const)) { + *const_bound = true; + } else if (lex_accept(T_volatile)) { + int depth = vd->type->ptr_level + vd->ptr_level; + if (depth < 32) + vd->pointer_volatile_mask |= 1U << depth; + } else { + lex_expect(T_restrict); + } + } +} -/* The closing parenthesis matching the opening one at @open. */ -static token_t *matching_close_bracket(token_t *open) +static int read_array_parameter_bound(var_t *vd, + bool is_param, + int dim, + bool *const_bound) { - int depth = 0; + bool static_bound = false; - for (; open; open = open->next) { - if (open->kind == T_open_bracket) - depth++; - else if (open->kind == T_close_bracket && !--depth) - return open; + read_array_parameter_qualifiers(vd, is_param, dim, &static_bound, + const_bound); + if (lex_peek(T_close_square, NULL)) { + if (static_bound) + error_at("static array parameter needs a bound", cur_token_loc()); + return 0; } - return NULL; + int bound = read_const_expr(vd->scope); + + if (static_bound && bound <= 0) + error_at("static array parameter needs a positive bound", + cur_token_loc()); + if (bound <= 0) + error_at("Array size must be positive", cur_token_loc()); + return bound; } +void read_inner_var_decl(var_t *vd, + bool anon, + bool is_param, + bool is_record_member); + /* Whether the declarator ahead declares a function returning a function * pointer, `(*get(void))(int)`, or a pointer to one, `(*(*pg)(void))(int)`: a * star inside a parenthesis followed by a parameter list, and another parameter @@ -705,20 +837,14 @@ static token_t *matching_close_bracket(token_t *open) static bool function_pointer_return_follows(void) { token_t *open = cur_token->next; - token_t *token; + token_t *token = function_pointer_declarator_suffix(open, NULL); token_t *close; - if (!open || open->kind != T_open_bracket || !(token = open->next) || - token->kind != T_asterisk) - return false; - while (token && (token->kind == T_asterisk || token->kind == T_const || - token->kind == T_volatile || token->kind == T_restrict)) - token = token->next; if (!token || !(token->kind == T_open_bracket || (token->kind == T_identifier && token->next && token->next->kind == T_open_bracket))) return false; - close = matching_close_bracket(open); + close = skip_balanced_delimiter(open, T_open_bracket, T_close_bracket); return close && close->next && close->next->kind == T_open_bracket; } @@ -734,14 +860,15 @@ static void read_function_pointer_return_declarator(var_t *vd, bool is_record_member) { token_t *start = cur_token; - token_t *close = matching_close_bracket(cur_token->next); + token_t *close = skip_balanced_delimiter(cur_token->next, T_open_bracket, + T_close_bracket); token_t *end; func_t *callback = arena_alloc_func(); type_t *callback_type = add_type(); int stars = 0; cur_token = close; - memcpy(&callback->return_def, vd, sizeof(var_t)); + set_function_return(&callback->return_def, vd); callback->return_def.var_name = NULL; callback->returns_aggregate = is_record_type(vd->type) && !has_effective_pointer(&callback->return_def); @@ -755,6 +882,7 @@ static void read_function_pointer_return_declarator(var_t *vd, callback_type->type_name[0] = '\0'; callback_type->ptr_level = 0; callback_type->pointer_const_mask = 0; + callback_type->pointer_volatile_mask = 0; callback_type->size = PTR_SIZE; callback_type->alignment = PTR_SIZE; callback_type->func_signature = callback; @@ -764,9 +892,7 @@ static void read_function_pointer_return_declarator(var_t *vd, lex_expect(T_open_bracket); while (lex_accept(T_asterisk)) { stars++; - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; + skip_type_qualifiers(); } /* `(**get(void))(int)` returns a pointer to such a callback, and each @@ -775,6 +901,7 @@ static void read_function_pointer_return_declarator(var_t *vd, vd->type = callback_type; vd->ptr_level = stars - 1; vd->pointer_const_mask = 0; + vd->pointer_volatile_mask = 0; vd->is_const_pointer = false; read_inner_var_decl(vd, anon, is_param, is_record_member); @@ -783,7 +910,7 @@ static void read_function_pointer_return_declarator(var_t *vd, func_t *func = arena_alloc_func(); type_t *function_type = add_type(); - memcpy(&func->return_def, vd, sizeof(var_t)); + set_function_return(&func->return_def, vd); read_parameter_list_decl(func, true); function_type->base_type = TYPE_typedef; function_type->size = PTR_SIZE; @@ -800,10 +927,10 @@ static void read_function_pointer_return_declarator(var_t *vd, cur_token = end; } -void read_inner_var_decl(var_t *vd, - bool anon, - bool is_param, - bool is_record_member) +static void read_inner_var_decl_body(var_t *vd, + bool anon, + bool is_param, + bool is_record_member) { bool grouped_name; @@ -825,6 +952,13 @@ void read_inner_var_decl(var_t *vd, vd->ptr_level = 0; } + if (lex_peek(T_asterisk, NULL) && vd->type && vd->type->func_signature && + !vd->type->is_direct_function_type && !vd->type->ptr_level && + !vd->ptr_level) { + vd->callback_is_volatile = vd->pointer_volatile_mask & 1U; + vd->pointer_volatile_mask &= ~1U; + } + /* `typedef int (*const cfn_t)(int)` makes a cfn_t object read-only, but * `cfn_t *p` only points to such a callback: its qualifier stays on the * type rather than on p's own pointer level. @@ -849,9 +983,11 @@ void read_inner_var_decl(var_t *vd, vd->is_const_pointer = true; if (vd->ptr_level <= 32) vd->pointer_const_mask |= 1U << (vd->ptr_level - 1); - } else if (lex_accept(T_volatile)) - vd->is_volatile = true; - else if (lex_accept(T_restrict)) + } else if (lex_accept(T_volatile)) { + int depth = vd->type->ptr_level + vd->ptr_level; + if (depth <= 32) + vd->pointer_volatile_mask |= 1U << (depth - 1); + } else if (lex_accept(T_restrict)) ; /* restrict is an aliasing contract, not storage state. */ else break; @@ -929,7 +1065,10 @@ void read_inner_var_decl(var_t *vd, else vd->parenthesized_function_pointer_inner_qualified = true; - vd->is_volatile = true; + int depth = + vd->type->ptr_level + vd->ptr_level + nested_ptr_level; + if (depth <= 32) + vd->pointer_volatile_mask |= 1U << (depth - 1); } else if (lex_accept(T_restrict)) { vd->parenthesized_function_pointer_restrict = true; if (nested_ptr_level >= 2) @@ -955,34 +1094,15 @@ void read_inner_var_decl(var_t *vd, * allocation can share their existing paths. */ for (int dim = 0; lex_accept(T_open_square); dim++) { - bool parameter_static_bound = false; bool parameter_const_bound = false; inner_array = true; if (dim >= 4) error_at("Array declarators support at most four dimensions", cur_token_loc()); - while (lex_peek(T_static, NULL) || lex_peek(T_const, NULL) || - lex_peek(T_volatile, NULL) || lex_peek(T_restrict, NULL)) { - if (!is_param || dim) - error_at("array bracket qualifiers require a parameter", - cur_token_loc()); - if (lex_accept(T_static)) { - if (parameter_static_bound) - error_at("duplicate static array parameter qualifier", - cur_token_loc()); - parameter_static_bound = true; - } else if (lex_accept(T_const)) - parameter_const_bound = true; - else if (lex_accept(T_volatile)) - vd->is_volatile = true; - else - lex_expect(T_restrict); - } - if (lex_peek(T_close_square, NULL)) { - if (parameter_static_bound) - error_at("static array parameter needs a bound", - cur_token_loc()); + int bound = read_array_parameter_bound(vd, is_param, dim, + ¶meter_const_bound); + if (!bound) { if (dim) error_at( "Only the outer function-pointer array bound may " @@ -990,24 +1110,9 @@ void read_inner_var_decl(var_t *vd, cur_token_loc()); vd->has_unsized_array = true; } else { - int bound = read_const_expr(vd->scope); - - if (parameter_static_bound && bound <= 0) - error_at("static array parameter needs a positive bound", - cur_token_loc()); - if (bound <= 0) - error_at("Array size must be positive", cur_token_loc()); - if (dim == 0) - vd->array_size = bound; - else { - if (dim == 1) - vd->array_dim2 = bound; - else if (dim == 2) - vd->array_dim3 = bound; - else if (dim == 3) - vd->array_dim4 = bound; - vd->array_size *= bound; - } + set_array_dimension(&vd->array_size, &vd->array_dim2, + &vd->array_dim3, &vd->array_dim4, dim, + bound, false); } if (parameter_const_bound && dim == 0) { vd->is_const_pointer = true; @@ -1022,7 +1127,7 @@ void read_inner_var_decl(var_t *vd, * from the pointer object's own storage extent. */ if (lex_peek(T_open_square, NULL)) { - int dims = 0; + fixed_array_shape_t pointee_shape = {0}; /* The nested stars make this a pointer-to-array. Preserve whether * its array element was plain void before adding them; a typedef or @@ -1043,33 +1148,16 @@ void read_inner_var_decl(var_t *vd, vd->is_const_pointer = vd->ptr_level <= 32 && (vd->pointer_const_mask & (1U << (vd->ptr_level - 1))); - while (lex_accept(T_open_square)) { - int bound; - - if (dims >= 4) - error_at( - "Array declarators support at most four dimensions", - cur_token_loc()); - if (lex_peek(T_close_square, NULL)) - error_at("Pointer-to-array needs a bound", cur_token_loc()); - bound = read_const_expr(vd->scope); - if (bound <= 0) - error_at("Array size must be positive", cur_token_loc()); - if (dims == 0) - vd->pointee_array_size = bound; - else { - if (dims == 1) - vd->pointee_array_dim2 = bound; - else if (dims == 2) - vd->pointee_array_dim3 = bound; - else - vd->pointee_array_dim4 = bound; - vd->pointee_array_size *= bound; - } - lex_expect(T_close_square); - dims++; - } - if (dims && void_array_element) + read_array_shape_bounds( + vd->scope, &pointee_shape, + "Array declarators support at most four dimensions", + "Pointer-to-array needs a bound", "Array size must be positive", + true, false); + fixed_array_shape_apply(&pointee_shape, &vd->pointee_array_size, + &vd->pointee_array_dim2, + &vd->pointee_array_dim3, + &vd->pointee_array_dim4); + if (pointee_shape.rank && void_array_element) error_at("void type cannot be an array element", cur_token_loc()); return; @@ -1129,7 +1217,7 @@ void read_inner_var_decl(var_t *vd, * declarator. Copy it into the syntax-only signature before the * function-pointer marker is set on vd. */ - memcpy(&func->return_def, vd, sizeof(var_t)); + set_function_return(&func->return_def, vd); func->returns_aggregate = is_record_type(func->return_def.type) && !has_effective_pointer(&func->return_def); read_parameter_list_decl(func, true); @@ -1141,6 +1229,8 @@ void read_inner_var_decl(var_t *vd, vd->func_signature = func; vd->is_func = true; vd->parenthesized_function_pointer_level = nested_ptr_level; + vd->callback_is_const = callback_const; + vd->callback_is_volatile = callback_volatile; if (nested_ptr_level >= 2) { /* `int (**slot)(int)` is an ordinary pointer object whose pointee * is the compact one-pointer callback representation. Keep that @@ -1157,8 +1247,6 @@ void read_inner_var_decl(var_t *vd, if (callback_restrict) error_at("restrict requires a pointer to an object type", cur_token_loc()); - vd->callback_is_const = callback_const; - vd->callback_is_volatile = callback_volatile; if (callback_volatile && !vd->parenthesized_function_pointer_outer_volatile) vd->is_volatile = false; @@ -1175,6 +1263,10 @@ void read_inner_var_decl(var_t *vd, vd->pointer_const_mask = (vd->pointer_const_mask & return_mask) | ((slot_mask >> 1) << return_depth); + slot_mask = vd->pointer_volatile_mask >> return_depth; + vd->pointer_volatile_mask = + (vd->pointer_volatile_mask & return_mask) | + ((slot_mask >> 1) << return_depth); } vd->ptr_level += nested_ptr_level - 1; vd->is_const_pointer = @@ -1262,36 +1354,14 @@ void read_inner_var_decl(var_t *vd, for (int dim = 0; lex_accept(T_open_square); dim++) { vd->has_direct_array_declarator = true; - bool parameter_static_bound = false; bool parameter_const_bound = false; if (dim >= 4) error_at("Array declarators support at most four dimensions", cur_token_loc()); - while (lex_peek(T_static, NULL) || lex_peek(T_const, NULL) || - lex_peek(T_volatile, NULL) || lex_peek(T_restrict, NULL)) { - if (!is_param || dim) - error_at("array bracket qualifiers require a parameter", - cur_token_loc()); - if (lex_accept(T_static)) { - if (parameter_static_bound || dim) - error_at( - "static is only valid in the outermost array " - "parameter bound", - cur_token_loc()); - parameter_static_bound = true; - } else if (lex_accept(T_const)) - parameter_const_bound = true; - else if (lex_accept(T_volatile)) - vd->is_volatile = true; - else - lex_expect(T_restrict); - } - if (lex_peek(T_close_square, NULL)) { - if (parameter_static_bound) - error_at("static array parameter needs a bound", - cur_token_loc()); - + int next_dim = read_array_parameter_bound(vd, is_param, dim, + ¶meter_const_bound); + if (!next_dim) { /* An omitted leading size is only a pointer when nothing * follows it: "int a[]" is "int *", but "int a[][4]" points at * rows of four and is indexed exactly like "int a[3][4]". @@ -1304,26 +1374,9 @@ void read_inner_var_decl(var_t *vd, else vd->ptr_level++; } else { - int next_dim = read_const_expr(vd->scope); - - if (parameter_static_bound && next_dim <= 0) - error_at("static array parameter needs a positive bound", - cur_token_loc()); - - if (dim == 0) { - vd->array_size = next_dim; - } else { - if (dim == 1) - vd->array_dim2 = next_dim; - else if (dim == 2) - vd->array_dim3 = next_dim; - else if (dim == 3) - vd->array_dim4 = next_dim; - if (vd->array_size > 0) - vd->array_size *= next_dim; - else - vd->array_size = next_dim; - } + set_array_dimension(&vd->array_size, &vd->array_dim2, + &vd->array_dim3, &vd->array_dim4, dim, + next_dim, true); } if (parameter_const_bound && dim == 0) { vd->is_const_pointer = true; @@ -1408,7 +1461,7 @@ void read_inner_var_decl(var_t *vd, bool saved_block_typedef_declarator = parsing_block_typedef_declarator; - memcpy(&func->return_def, vd, sizeof(var_t)); + set_function_return(&func->return_def, vd); func->returns_aggregate = is_record_type(func->return_def.type) && !has_effective_pointer(&func->return_def); @@ -1494,6 +1547,27 @@ void read_inner_var_decl(var_t *vd, error_at("void type cannot define an object", cur_token_loc()); } +/* The element count of a declarator is bounded as its suffixes are read, but + * size_var() scales it by the element size afterwards, which can wrap as well: + * `struct big a[4]` with a member array of a gigabyte. Check the extent once + * the declarator is complete, before layout or allocation relies on it. + */ +void read_inner_var_decl(var_t *vd, + bool anon, + bool is_param, + bool is_record_member) +{ + int element; + + read_inner_var_decl_body(vd, anon, is_param, is_record_member); + if (vd->array_size <= 0 || !vd->type) + return; + element = array_element_size(vd); + if (!element && vd->type->base_struct) + element = vd->type->base_struct->size; + reject_oversized_array(vd->array_size, element); +} + /* C99 6.7p2 lets the storage-class specifiers appear anywhere among the * declaration specifiers: "int static x;" is "static int x;" and "unsigned * register int z;" is "register unsigned int z;". The order carries no meaning, @@ -1620,7 +1694,7 @@ type_t *read_record_body(block_t *parent, int max_size = 0; bitfield_layout_t bits = {0}; bool has_flexible_array_member = false; - block_t *scope = parent ? parent : GLOBAL_BLOCK; + block_t *scope = parent ? parent : CURRENT_TU_SCOPE; if (has_tag) { type = local_record_tag(token, scope, kind); @@ -1652,6 +1726,14 @@ type_t *read_record_body(block_t *parent, else reject_flexible_array_member_container(last); mark_flexible_array_member(last, is_union); + + /* A member needs a complete type to have a size, and a record is + * not complete until its closing brace, so `struct s { struct s x; + * }` is invalid as well. Pointers to it remain valid. A flexible + * array has no size of its own, but its elements still need one. + */ + if (is_incomplete_record_object(last)) + error_at("Record member has incomplete type", cur_token_loc()); if (is_union) { last->offset = 0; int field_size = @@ -1696,10 +1778,10 @@ type_t *read_record_body(block_t *parent, } } while (!lex_accept(T_close_curly)); + if (!is_union) + max_size = flush_bitfield_layout(size, &bits); type->alignment = alignment; - type->size = is_union - ? ALIGN_UP(max_size, alignment) - : ALIGN_UP(flush_bitfield_layout(size, &bits), alignment); + type->size = align_record_offset(max_size, alignment); type->num_fields = i; type->is_union = is_union; type->has_flexible_array_member = has_flexible_array_member; @@ -1764,6 +1846,7 @@ void read_full_var_decl(var_t *vd, vd->pointee_func_signature = type->pointee_func_signature; vd->is_func = vd->func_signature != NULL; vd->is_const_qualified = type->is_const_qualified; + vd->pointer_volatile_mask = type->pointer_volatile_mask; if (type->func_signature && !type->is_direct_function_type && (type->pointer_const_mask & 1U)) { vd->is_const_pointer = true; @@ -1777,9 +1860,8 @@ void read_full_var_decl(var_t *vd, if (type->pointee_array_size) { fixed_array_shape_t shape = fixed_array_shape_from_pointee_type(type); - fixed_array_shape_to_pointee_var(vd, &shape); - vd->pointee_array_element_ptr_level = - type->pointee_array_element_ptr_level; + set_pointee_array_shape(vd, &shape, + type->pointee_array_element_ptr_level); } if (type->ptr_level && type->ptr_level <= 32) vd->is_const_pointer = @@ -1814,8 +1896,16 @@ void read_full_var_decl(var_t *vd, break; } vd->is_inline = is_inline; - vd->is_volatile = - declaration_volatile || is_volatile || type->is_volatile_qualified; + vd->is_volatile = type->is_volatile_qualified; + if (declaration_volatile || is_volatile) { + int depth = type->ptr_level + (type->func_signature && + !type->is_direct_function_type && + !type->ptr_level); + if (depth && depth <= 32) + vd->pointer_volatile_mask |= 1U << (depth - 1); + else + vd->is_volatile = true; + } if (is_const || declaration_const) { if (type->ptr_level) { vd->is_const_pointer = true; @@ -1825,6 +1915,9 @@ void read_full_var_decl(var_t *vd, vd->is_const_qualified = true; } + vd->type = qualify_type_name(type, is_const || declaration_const, + is_volatile || declaration_volatile); + read_inner_var_decl(vd, anon, is_param, is_record_member); if (!parsing_sizeof_function_signature && vd->func_signature && @@ -1842,7 +1935,19 @@ void read_full_var_decl(var_t *vd, /* starting next_token, need to check the type */ void read_partial_var_decl(var_t *vd, var_t *template) { - UNUSED(template); + int depth = vd->type->ptr_level + (vd->type->func_signature && + !vd->type->is_direct_function_type && + !vd->type->ptr_level); + vd->pointer_volatile_mask = vd->type->pointer_volatile_mask; + if (depth && depth <= 32) { + if (template) + vd->pointer_volatile_mask |= + template->pointer_volatile_mask & + (depth == 32 ? ~0U : (1U << depth) - 1); + else if (vd->is_volatile) + vd->pointer_volatile_mask |= 1U << (depth - 1); + vd->is_volatile = vd->type->is_volatile_qualified; + } read_inner_var_decl(vd, false, false, false); } @@ -1985,8 +2090,7 @@ void read_literal_param(block_t *parent, basic_block_t *bb) int length = read_concatenated_string(combined); const int index = write_string_symbol(combined, length); - var_t *vd = require_typed_ptr_var(parent, TY_char, true); - vd->var_name = gen_name(); + var_t *vd = name_var(require_typed_ptr_var(parent, TY_char, true)); vd->init_val = index; vd->is_const_qualified = true; vd->is_string_literal = true; @@ -2002,183 +2106,28 @@ void read_literal_param(block_t *parent, basic_block_t *bb) */ void parse_string_array_init(var_t *var, block_t *parent, basic_block_t **bb) { - char combined[MAX_STRING_LEN]; - int len; - int count; - - len = read_concatenated_string(combined) + 1; - if (var->has_unsized_array) { - var->array_size = len; - var->has_unsized_array = false; - } else if (len - 1 > var->array_size) - error_at("String initializer is too long for character array", - cur_token_loc()); - - /* The terminating null is dropped when the array has room only for the - * characters (C99 6.7.8p14), so never store past the array. - */ - if (len > var->array_size) - len = var->array_size; - - /* Elements past the string are zero. Static storage already starts out - * zeroed, but an automatic array is reinitialized on every entry to its - * declaration and must clear whatever the slot held before. - */ - count = parent == GLOBAL_BLOCK ? len : var->array_size; - for (int i = 0; i < count; i++) { - var_t *value = require_var(parent); - var_t *addr; - - value->var_name = gen_name(); - value->init_val = i < len ? (unsigned char) combined[i] : 0; - value->is_const = true; - add_insn(parent, *bb, OP_load_constant, value, NULL, NULL, 0, NULL); - addr = compute_element_address(parent, bb, var, i, 1); - add_insn(parent, *bb, OP_write, NULL, addr, value, 1, NULL); - } + parse_string_array_init_to(var, parent, bb, var, var, false); } void parse_wstring_array_init(var_t *var, block_t *parent, basic_block_t **bb) { - int values[MAX_STRING_LEN]; - int length; - int units; - - length = read_wstring_units(values, MAX_STRING_LEN); - - /* As for a char array, the terminating null may be dropped (C99 6.7.8p15). - */ - units = length + 1; - if (var->has_unsized_array) { - var->array_size = units; - var->has_unsized_array = false; - } else if (length > var->array_size) - error_at("Wide string initializer is too long for array", - cur_token_loc()); - - for (int i = 0; i < var->array_size; i++) { - var_t *value = require_typed_var(parent, var->type); - var_t *addr; - - value->var_name = gen_name(); - value->init_val = i < length ? values[i] : 0; - value->is_const = true; - add_insn(parent, *bb, OP_load_constant, value, NULL, NULL, 0, NULL); - addr = compute_element_address(parent, bb, var, i, var->type->size); - add_insn(parent, *bb, OP_write, NULL, addr, value, var->type->size, - NULL); - } -} - -bool numeric_has_unsigned_suffix(const char *token) -{ - for (int i = 0; token[i]; i++) - if ((token[i] | 32) == 'u') - return true; - return false; -} - -/* C99 permits U, L, LL, UL, ULL, LU, and LLU (case-insensitively). Keep this - * separate from type selection: malformed suffixes must not become a valid wide - * literal merely because their letters happen to be counted. - */ -bool numeric_suffix_is_valid(const char *suffix) -{ - int pos = 0; - - if ((suffix[pos] | 32) == 'u') - pos++; - if ((suffix[pos] | 32) == 'l') { - pos++; - if ((suffix[pos] | 32) == 'l') - pos++; - } - if ((suffix[pos] | 32) == 'u') - pos++; - return suffix[pos] == '\0'; + parse_string_array_init_to(var, parent, bb, var, var, true); } -bool numeric_has_long_long_suffix(const char *token) -{ - int long_suffix_count = 0; - - for (int i = 0; token[i]; i++) - if ((token[i] | 32) == 'l') - long_suffix_count++; - return long_suffix_count == 2; -} +typedef struct { + int unsigned_count; + int long_count; +} numeric_suffix_info_t; -int numeric_long_suffix_count(const char *token) +static numeric_suffix_info_t numeric_suffix_info(const char *token) { - int count = 0; + numeric_suffix_info_t info = {0}; - for (int i = 0; token[i]; i++) - if ((token[i] | 32) == 'l') - count++; - return count; -} - -/* The file-scope constant evaluator is still word-sized. Decide from the token - * spelling whether it must use the two-word literal path before that evaluator - * consumes and narrows it. - */ -bool numeric_literal_needs_wide_path(const char *token) -{ - const char *digits = token; - int count = 0; - bool is_unsigned = numeric_has_unsigned_suffix(token); - - if (numeric_has_long_long_suffix(token)) - return true; - if (digits[0] == '0' && (digits[1] | 32) == 'x') { - digits += 2; - while (isxdigit(digits[count])) - count++; - return count > 8; - } - if (digits[0] == '0' && (digits[1] | 32) == 'b') { - digits += 2; - while (digits[count] == '0' || digits[count] == '1') - count++; - return count > 32; + for (int i = 0; token[i]; i++) { + info.unsigned_count += (token[i] | 32) == 'u'; + info.long_count += (token[i] | 32) == 'l'; } - if (digits[0] == '0') { - while (digits[count] >= '0' && digits[count] <= '7') - count++; - return count > 12 || - (count == 12 && strncmp(digits, "037777777777", count) > 0); - } - while (isdigit(digits[count])) - count++; - if (count > 10) - return true; - if (count < 10) - return false; - return strncmp(digits, is_unsigned ? "4294967295" : "2147483647", count) > - 0; -} - -/* Some bootstrap stages still materialize the exact decimal 2^31 token as an - * int bit pattern before the global initializer path sees it. This path was - * selected from the original token spelling, so restore the required wide - * candidate type before phase-2 chooses its constant-load width. - */ -void force_wide_global_literal_type(var_t *value, const char *token) -{ - /* The typed global path also handles a one-word unsigned literal such as - * 0xffffffff: it must retain TY_uint for shifts, comparisons, and division, - * but it is not a long-long candidate. Promoting it here made every 32-bit - * target reject a valid C99 unsigned-int initializer merely because that - * path was selected. - */ - if (!numeric_literal_needs_wide_path(token)) - return; - if (value->type->size >= 8) - return; - value->type = numeric_has_unsigned_suffix(token) || - (unsigned int) value->init_val_hi > 0x7fffffffU - ? TY_ulong_long - : TY_long_long; + return info; } /* Accumulate a 64-bit token in four 16-bit limbs. Each intermediate stays small @@ -2209,51 +2158,97 @@ bool numeric_mul_add_wide(unsigned int *hi, return true; } -/* The global-initializer fast path historically carries only an int value. - * Route literals whose C99 candidate is unsigned through the typed path even - * when they fit in one word: otherwise 0xffffffff is folded as -1 before a - * right shift, comparison, or division sees its unsigned rank. - */ -bool numeric_literal_needs_typed_global_path(const char *token) +typedef struct { + int base; + int digits; + const char *digit_text; + bool is_unsigned; + bool decimal; + bool needs_wide; + bool needs_typed_global; +} numeric_literal_info_t; + +static numeric_literal_info_t numeric_literal_info(const char *token, + bool include_typed) { + numeric_suffix_info_t suffix = numeric_suffix_info(token); + numeric_literal_info_t info = {10, 0, token, false, true, false, false}; unsigned int hi = 0; unsigned int lo = 0; - int i = 0; - int base = 10; - bool is_decimal = true; - if (numeric_has_unsigned_suffix(token)) - return true; - if (token[0] == '0') { - if ((token[1] | 32) == 'x') { - i = 2; - base = 16; - is_decimal = false; - while (isxdigit(token[i])) { - char digit = token[i++]; - - if (isdigit(digit)) - digit -= '0'; - else - digit = (digit | 32) - 'a' + 10; - numeric_mul_add_wide(&hi, &lo, base, digit); - } - } else if ((token[1] | 32) == 'b') { - i = 2; - base = 2; - is_decimal = false; - while (token[i] == '0' || token[i] == '1') - numeric_mul_add_wide(&hi, &lo, base, token[i++] - '0'); - } else { - base = 8; - is_decimal = false; - while (token[i] >= '0' && token[i] <= '7') - numeric_mul_add_wide(&hi, &lo, base, token[i++] - '0'); - } + info.is_unsigned = suffix.unsigned_count != 0; + if (suffix.long_count == 2) { + info.needs_wide = true; + info.needs_typed_global = include_typed && info.is_unsigned; + return info; } - if (is_decimal) - return false; - return hi != 0 || lo > 0x7fffffffU; + info.base = numeric_literal_base(token); + info.decimal = info.base == 10; + if (info.base == 16 || info.base == 2) + info.digit_text += 2; + int max_typed_digits = info.base == 16 ? 8 : info.base == 2 ? 32 : 12; + while (info.digit_text[info.digits]) { + char c = info.digit_text[info.digits] | 32; + unsigned int digit = isdigit(c) ? c - '0' : c - 'a' + 10; + + if ((!isdigit(c) && !isxdigit(c)) || digit >= (unsigned int) info.base) + break; + if (include_typed && !info.decimal && !info.is_unsigned && + info.digits < max_typed_digits) + numeric_mul_add_wide(&hi, &lo, info.base, digit); + info.digits++; + } + if (info.base == 16) + info.needs_wide = info.digits > 8; + else if (info.base == 2) + info.needs_wide = info.digits > 32; + else if (info.base == 8) + info.needs_wide = info.digits > 12 || + (info.digits == 12 && + strncmp(token, "037777777777", info.digits) > 0); + else + info.needs_wide = + info.digits > 10 || + (info.digits == 10 && + strncmp(token, info.is_unsigned ? "4294967295" : "2147483647", + info.digits) > 0); + info.needs_typed_global = + include_typed && + (info.is_unsigned || (!info.decimal && (hi || lo > 0x7fffffffU))); + return info; +} + +/* Keep token routing in one classifier: the word-sized evaluator must preserve + * unsigned candidates and route wide values before narrowing them. + */ +bool numeric_literal_needs_wide_path(const char *token) +{ + return numeric_literal_info(token, false).needs_wide; +} + +bool numeric_literal_needs_global_reader(const char *token) +{ + numeric_literal_info_t info = numeric_literal_info(token, true); + + return info.needs_wide || info.needs_typed_global; +} + +/* Some bootstrap stages materialize decimal 2^31 as an int bit pattern. Restore + * its wide candidate type before phase 2 chooses its constant-load width. + */ +void force_wide_global_literal_type(var_t *value, const char *token) +{ + numeric_literal_info_t literal = numeric_literal_info(token, false); + + /* The global-reader path also handles unsigned int values such as + * 0xffffffff, which this helper must not promote to long long. + */ + if (!literal.needs_wide || value->type->size >= 8) + return; + value->type = + literal.is_unsigned || (unsigned int) value->init_val_hi > 0x7fffffffU + ? TY_ulong_long + : TY_long_long; } void read_numeric_param(block_t *parent, basic_block_t *bb, bool is_neg) @@ -2268,11 +2263,13 @@ void read_numeric_param(block_t *parent, basic_block_t *bb, bool is_neg) bool has_unsigned_suffix; int long_suffix_count; bool is_long_long; + numeric_suffix_info_t suffix; lex_ident_n(T_numeric, token, MAX_TOKEN_LEN); - has_unsigned_suffix = numeric_has_unsigned_suffix(token); - long_suffix_count = numeric_long_suffix_count(token); - is_long_long = long_suffix_count >= 2; + suffix = numeric_suffix_info(token); + has_unsigned_suffix = suffix.unsigned_count != 0; + long_suffix_count = suffix.long_count; + is_long_long = suffix.long_count >= 2; if (token[0] == '-') { is_neg = !is_neg; @@ -2357,8 +2354,7 @@ void read_numeric_param(block_t *parent, basic_block_t *bb, bool is_neg) !has_unsigned_suffix && !long_suffix_count && !value_hi && value == 0x80000000U; - var_t *vd = require_var(parent); - vd->var_name = gen_name(); + var_t *vd = require_named_var(parent); if (narrow_int_min) { vd->type = TY_int; } else if (is_long_long || value_hi || @@ -2395,17 +2391,13 @@ void read_numeric_param(block_t *parent, basic_block_t *bb, bool is_neg) void read_char_param(block_t *parent, basic_block_t *bb) { - char literal[MAX_TOKEN_LEN], unescaped[MAX_TOKEN_LEN]; + char literal[MAX_TOKEN_LEN]; lex_ident(T_char, literal); - unescape_string(literal, unescaped, MAX_TOKEN_LEN); - - var_t *vd = require_typed_var(parent, TY_int); - vd->var_name = gen_name(); - vd->init_val = parse_character_constant(literal); + int value = parse_character_constant(literal); + var_t *vd = load_constant(parent, bb, value, TY_int); vd->is_const = true; opstack_push(vd); - add_insn(parent, bb, OP_load_constant, vd, NULL, NULL, 0, NULL); } /* The current execution wide-character representation is int. Decode the same @@ -2414,17 +2406,13 @@ void read_char_param(block_t *parent, basic_block_t *bb) */ void read_wchar_param(block_t *parent, basic_block_t *bb) { - char literal[MAX_TOKEN_LEN], unescaped[MAX_TOKEN_LEN]; + char literal[MAX_TOKEN_LEN]; lex_ident(T_wchar, literal); - unescape_string(literal, unescaped, MAX_TOKEN_LEN); - - var_t *vd = require_typed_var(parent, TY_int); - vd->var_name = gen_name(); - vd->init_val = parse_wide_character_constant(literal); + int value = parse_wide_character_constant(literal); + var_t *vd = load_constant(parent, bb, value, TY_int); vd->is_const = true; opstack_push(vd); - add_insn(parent, bb, OP_load_constant, vd, NULL, NULL, 0, NULL); } void read_wstring_param(block_t *parent, basic_block_t *bb) @@ -2432,8 +2420,8 @@ void read_wstring_param(block_t *parent, basic_block_t *bb) int values[MAX_STRING_LEN]; int length = read_wstring_units(values, MAX_STRING_LEN); - var_t *vd = require_typed_ptr_var(parent, find_type("wchar_t", true), 1); - vd->var_name = gen_name(); + var_t *vd = name_var( + require_typed_ptr_var(parent, find_builtin_type("wchar_t"), 1)); vd->init_val = write_wide_symbol(values, length); vd->is_const_qualified = true; vd->is_string_literal = true; @@ -2441,83 +2429,16 @@ void read_wstring_param(block_t *parent, basic_block_t *bb) add_insn(parent, bb, OP_load_rodata_address, vd, NULL, NULL, 0, NULL); } -/* Whether the static initializer at the next token is a string literal plus or - * minus an integer constant, as in `"abc" + 1` or `2 + "abcd"`: an address - * constant (C99 6.6p7). The literal must follow a '+' or start the initializer - * with an additive operator after it; a subscripted one is an element instead. - */ -bool string_address_offset_starts_here(void) -{ - token_t *prev = NULL; - int bracket_depth = 0; - - for (token_t *token = cur_token->next; token; token = token->next) { - if (token->kind == T_open_bracket) - bracket_depth++; - else if (token->kind == T_close_bracket) { - if (bracket_depth == 0) - return false; - bracket_depth--; - } else if (bracket_depth == 0 && - (token->kind == T_semicolon || token->kind == T_comma || - token->kind == T_close_curly)) - return false; - else if (bracket_depth == 0 && - (token->kind == T_string || token->kind == T_wstring)) { - token_t *after = token; - - while (after->next && after->next->kind == token->kind) - after = after->next; - after = after->next; - if (after && after->kind == T_open_square) - return false; - if (!prev) - return after && - (after->kind == T_plus || after->kind == T_minus); - return prev->kind == T_plus; - } - prev = token; - } - return false; -} - -/* The address constant string_address_offset_starts_here() accepted: the - * literal's address advanced by the integer constants added to or subtracted - * from it, in element units. - */ -var_t *read_string_address_offset(block_t *parent, - basic_block_t *bb, - block_t *scope) +/* Advance a string literal address by an element index. */ +static var_t *string_literal_address(block_t *parent, + basic_block_t **bb, + var_t *literal, + int index, + int stride) { - var_t *literal; - var_t *address; - int index = 0; - - if (!lex_peek(T_string, NULL) && !lex_peek(T_wstring, NULL)) { - token_t *token = cur_token->next; - token_t *plus = NULL; + var_t *address = + compute_element_address(parent, bb, literal, index, stride); - /* Find the '+' before the literal and end the integer operand there - * while it is read, so the constant reader does not meet the literal. - */ - while (token->kind != T_string && token->kind != T_wstring) { - plus = token; - token = token->next; - } - plus->kind = T_semicolon; - index = read_const_expr(scope); - plus->kind = T_plus; - lex_expect(T_plus); - } - if (lex_peek(T_wstring, NULL)) - read_wstring_param(parent, bb); - else - read_literal_param(parent, bb); - literal = opstack_pop(); - if (lex_peek(T_plus, NULL) || lex_peek(T_minus, NULL)) - index += read_global_address_offset(scope, parent, bb); - address = compute_element_address(parent, &bb, literal, index, - literal->type->size); if (address != literal) { address->type = literal->type; address->ptr_level = literal->ptr_level; @@ -2536,7 +2457,6 @@ var_t *read_string_literal_element_address(block_t *parent, { token_t *literal_token = cur_token; var_t *literal; - var_t *address; int index; /* Check the index against the literal, then read the literal again to emit @@ -2552,13 +2472,6 @@ var_t *read_string_literal_element_address(block_t *parent, lex_expect(T_open_square); read_const_expr(scope); lex_expect(T_close_square); - address = compute_element_address(parent, &bb, literal, index, - literal->type->size); - if (address != literal) { - address->type = literal->type; - address->ptr_level = literal->ptr_level; - address->is_const_qualified = true; - address->is_string_literal = true; - } - return address; + return string_literal_address(parent, &bb, literal, index, + literal->type->size); } diff --git a/src/parser-expr.c b/src/parser-expr.c index 508ec68d..60080957 100644 --- a/src/parser-expr.c +++ b/src/parser-expr.c @@ -12,279 +12,79 @@ * definition that precedes it there and cannot be compiled on its own. */ -/* The grouped row forms below accept only `(*name)` where name points to a - * scalar fixed array; any other operand, such as `(*pp)[1]` on an int **, - * belongs to ordinary expression parsing. - */ -static bool names_scalar_row_pointer(token_t *token, block_t *parent) -{ - var_t *var = find_var(token->literal, parent); - - return var && var->pointee_array_size && - !var->pointee_array_element_ptr_level && - effective_pointer_depth(var) == 1; -} - -/* Keep grouped row-element postfix support deliberately exact until general - * grouped lvalues have an address representation. This scanner never consumes - * tokens, so every non-match remains owned by ordinary expression parsing. - */ -static bool grouped_scalar_pointee_row_postfix_starts(block_t *parent) -{ - token_t *token = cur_token ? cur_token->next : NULL; - int suffixes = 0; - - if (!token || token->kind != T_open_bracket || !(token = token->next) || - token->kind != T_asterisk || !(token = token->next) || - token->kind != T_identifier || - !names_scalar_row_pointer(token, parent) || !(token = token->next) || - token->kind != T_close_bracket || !(token = token->next)) - return false; - - while (suffixes < 4 && token && token->kind == T_open_square) { - int depth = 0; - - for (; token; token = token->next) { - if (token->kind == T_open_square || token->kind == T_open_bracket || - token->kind == T_open_curly) - depth++; - else if (token->kind == T_close_square || - token->kind == T_close_bracket || - token->kind == T_close_curly) { - if (!--depth) { - token = token->next; - break; - } - } - } - if (!token) - return false; - suffixes++; - } - - return suffixes && - (token->kind == T_increment || token->kind == T_decrement); -} +static var_t *combine_assignment_value(block_t *parent, + basic_block_t **bb, + opcode_t op, + var_t *current, + var_t *value); +static var_t *combine_addressed_assignment(block_t *parent, + basic_block_t **bb, + opcode_t op, + var_t *address, + const lvalue_t *target, + var_t *value); +static var_t *store_addressed_assignment(block_t *parent, + basic_block_t **bb, + var_t *address, + const lvalue_t *target, + var_t *value, + bool use_resize_to); +static bool accept_assignment_op(opcode_t *op); -static void handle_grouped_scalar_pointee_row_postfix(block_t *parent, - basic_block_t **bb) +static opcode_t read_increment_operator(void) { - char name[MAX_VAR_LEN]; - var_t *source, *row, *index, *address, *old, *one, *updated; - fixed_array_shape_t shape; - int dimensions; - int depth = 0; - opcode_t op; - - lex_expect(T_open_bracket); - lex_expect(T_asterisk); - lex_ident(T_identifier, name); - source = find_var(name, parent); - if (!source) - error_at("Undeclared identifier", cur_token_loc()); - if (!lower_pointee_array_dereference(source, parent, bb)) - error_at("Grouped pointer-to-array update requires a scalar fixed row", - cur_token_loc()); - row = opstack_pop(); - if (row->is_const_qualified) - error_at("assignment of read-only location", cur_token_loc()); - shape = fixed_array_shape_from_var(row); - dimensions = shape.rank; - if (dimensions >= 3 && - (is_record_type(row->type) || row->type->is_floating)) - error_at( - "Grouped rank-three/four postfix update requires an integer scalar", - cur_token_loc()); - if (dimensions > 4) - error_at( - "Grouped pointer-to-array postfix update supports at most four " - "dimensions", - cur_token_loc()); - lex_expect(T_close_bracket); - address = row; - while (lex_accept(T_open_square)) { - int stride; - - if (depth >= dimensions) - error_at("Too many grouped array subscripts", cur_token_loc()); - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - index = opstack_pop(); - lex_expect(T_close_square); - stride = fixed_array_shape_stride(&shape, depth, row->type->size); - if (stride != 1) { - one = require_var(parent); - one->var_name = gen_name(); - one->init_val = stride; - add_insn(parent, *bb, OP_load_constant, one, NULL, NULL, 0, NULL); - updated = require_var(parent); - updated->var_name = gen_name(); - add_insn(parent, *bb, OP_mul, updated, index, one, 0, NULL); - index = updated; - } - updated = require_typed_ptr_var(parent, row->type, 1); - updated->var_name = gen_name(); - add_insn(parent, *bb, OP_add, updated, address, index, 0, NULL); - address = updated; - depth++; - } - if (depth != dimensions) - error_at("Grouped pointer-to-array postfix update needs all subscripts", - cur_token_loc()); - old = require_typed_var(parent, row->type); - old->var_name = gen_name(); - add_insn(parent, *bb, OP_read, old, address, NULL, row->type->size, NULL); if (lex_accept(T_increment)) - op = OP_add; - else { - lex_expect(T_decrement); - op = OP_sub; - } - one = require_typed_var(parent, TY_int); - one->var_name = gen_name(); - one->init_val = 1; - add_insn(parent, *bb, OP_load_constant, one, NULL, NULL, 0, NULL); - updated = require_var(parent); - updated->var_name = gen_name(); - updated->type = integer_binary_result_type(op, old, one); - add_insn(parent, *bb, op, updated, old, one, 0, NULL); - updated = resize_to(parent, bb, updated, row->type, 0); - add_insn(parent, *bb, OP_write, NULL, address, updated, row->type->size, - NULL); - opstack_push(old); + return OP_add; + lex_expect(T_decrement); + return OP_sub; } -/* Keep prefix support equally narrow: the generic prefix path has no general - * grouped-lvalue address representation. This scanner is non-mutating so a - * non-match remains entirely owned by that generic path. - */ -static bool grouped_scalar_pointee_row_prefix_starts(block_t *parent) +static var_t *scale_pointer_index(block_t *parent, + basic_block_t *bb, + var_t *index, + int stride, + bool rematerialize_scale) { - token_t *token = cur_token ? cur_token->next : NULL; - int suffixes = 0; - - if (!token || (token->kind != T_increment && token->kind != T_decrement) || - !(token = token->next) || token->kind != T_open_bracket || - !(token = token->next) || token->kind != T_asterisk || - !(token = token->next) || token->kind != T_identifier || - !names_scalar_row_pointer(token, parent) || !(token = token->next) || - token->kind != T_close_bracket || !(token = token->next)) - return false; + long long scaled; + var_t *scale; + var_t *result; - while (suffixes < 4 && token && token->kind == T_open_square) { - int depth = 0; - - for (; token; token = token->next) { - if (token->kind == T_open_square || token->kind == T_open_bracket || - token->kind == T_open_curly) - depth++; - else if (token->kind == T_close_square || - token->kind == T_close_bracket || - token->kind == T_close_curly) { - if (!--depth) { - token = token->next; - break; - } - } - } - if (!token) - return false; - suffixes++; - } - return suffixes; + if (stride == 1) + return index; + type_t *index_type = integer_promoted_type(index); + if (index_type->size < PTR_SIZE) + index = promote_unchecked( + parent, &bb, index, + index_type->is_unsigned ? TY_ulong_long : TY_long_long, 0); + scaled = (long long) index->init_val * stride; + if (index->is_const && !index->is_global && index->type && + index->type->size <= TY_int->size && scaled >= INT_MIN && + scaled <= INT_MAX) + return load_constant(parent, bb, (int) scaled, + integer_promoted_type(index)); + scale = load_constant(parent, bb, stride, TY_int); + scale->is_const = rematerialize_scale; + result = name_var(require_typed_var(parent, integer_promoted_type(index))); + add_insn(parent, bb, OP_mul, result, index, scale, 0, NULL); + return result; } -static void handle_grouped_scalar_pointee_row_prefix(block_t *parent, - basic_block_t **bb) +static var_t *lower_array_element_address(block_t *parent, + basic_block_t *bb, + var_t *base, + var_t *index, + type_t *element_type, + int pointer_level, + int stride, + bool rematerialize_scale) { - char name[MAX_VAR_LEN]; - var_t *source, *row, *index, *address, *current, *one, *updated; - fixed_array_shape_t shape; - int dimensions; - int depth = 0; - opcode_t op; - - if (lex_accept(T_increment)) - op = OP_add; - else { - lex_expect(T_decrement); - op = OP_sub; - } - lex_expect(T_open_bracket); - lex_expect(T_asterisk); - lex_ident(T_identifier, name); - source = find_var(name, parent); - if (!source) - error_at("Undeclared identifier", cur_token_loc()); - if (!lower_pointee_array_dereference(source, parent, bb)) - error_at("Grouped pointer-to-array update requires a scalar fixed row", - cur_token_loc()); - row = opstack_pop(); - if (row->is_const_qualified) - error_at("assignment of read-only location", cur_token_loc()); - shape = fixed_array_shape_from_var(row); - dimensions = shape.rank; - if (dimensions >= 3 && - (is_record_type(row->type) || row->type->is_floating)) - error_at( - "Grouped rank-three/four prefix update requires an integer scalar", - cur_token_loc()); - if (dimensions > 4) - error_at( - "Grouped pointer-to-array prefix update supports at most four " - "dimensions", - cur_token_loc()); - lex_expect(T_close_bracket); - address = row; - while (lex_accept(T_open_square)) { - int stride; + var_t *address; - if (depth >= dimensions) - error_at("Too many grouped array subscripts", cur_token_loc()); - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - index = opstack_pop(); - lex_expect(T_close_square); - stride = fixed_array_shape_stride(&shape, depth, row->type->size); - if (stride != 1) { - one = require_var(parent); - one->var_name = gen_name(); - one->init_val = stride; - add_insn(parent, *bb, OP_load_constant, one, NULL, NULL, 0, NULL); - updated = require_var(parent); - updated->var_name = gen_name(); - add_insn(parent, *bb, OP_mul, updated, index, one, 0, NULL); - index = updated; - } - updated = require_typed_ptr_var(parent, row->type, 1); - updated->var_name = gen_name(); - add_insn(parent, *bb, OP_add, updated, address, index, 0, NULL); - address = updated; - depth++; - } - if (depth != dimensions) - error_at("Grouped pointer-to-array prefix update needs all subscripts", - cur_token_loc()); - current = require_typed_var(parent, row->type); - current->var_name = gen_name(); - add_insn(parent, *bb, OP_read, current, address, NULL, row->type->size, - NULL); - one = require_typed_var(parent, TY_int); - one->var_name = gen_name(); - one->init_val = 1; - add_insn(parent, *bb, OP_load_constant, one, NULL, NULL, 0, NULL); - updated = require_var(parent); - updated->var_name = gen_name(); - updated->type = integer_binary_result_type(op, current, one); - add_insn(parent, *bb, op, updated, current, one, 0, NULL); - updated = resize_to(parent, bb, updated, row->type, 0); - add_insn(parent, *bb, OP_write, NULL, address, updated, row->type->size, - NULL); - opstack_push(updated); + index = scale_pointer_index(parent, bb, index, stride, rematerialize_scale); + address = + name_var(require_typed_ptr_var(parent, element_type, pointer_level)); + add_insn(parent, bb, OP_add, address, base, index, 0, NULL); + return address; } /* The type name read after "(" by read_parenthesized_operand(), for the cast or @@ -317,6 +117,92 @@ typedef struct { int func_pointer_level; } paren_type_name_t; +/* The explicit array declarators in local and file-scope compound literals + * share one grammar. The caller maps this shape to expression or storage + * metadata after parsing. + */ +static bool read_compound_array_shape(block_t *scope, + int pointer_level, + bool file_scope, + paren_type_name_t *shape) +{ + if (lex_accept(T_open_bracket)) { + if (pointer_level || !lex_peek(T_asterisk, NULL)) + error_at("Array compound literal needs a pointer declarator", + cur_token_loc()); + do { + lex_expect(T_asterisk); + shape->array_element_ptr_level++; + skip_type_qualifiers(); + } while (lex_peek(T_asterisk, NULL)); + lex_expect(T_open_square); + if (!lex_peek(T_close_square, NULL)) { + shape->array_size = read_const_expr(scope); + if (shape->array_size <= 0) + error_at("Array compound literal needs a positive bound", + cur_token_loc()); + } else { + shape->array_outer_unsized = true; + } + lex_expect(T_close_square); + lex_expect(T_close_bracket); + fixed_array_shape_t pointee_shape = {0}; + + read_array_shape_bounds( + scope, &pointee_shape, + "Array declarators support at most four dimensions", NULL, + "Array size must be positive", false, false); + fixed_array_shape_apply(&pointee_shape, &shape->pointee_array_size, + &shape->pointee_array_dim2, + &shape->pointee_array_dim3, + &shape->pointee_array_dim4); + + if (!file_scope && !pointee_shape.rank) + error_at("Array compound literal needs a row bound", + cur_token_loc()); + shape->array_dims = 1; + shape->parenthesized_array = true; + return true; + } + + while (lex_accept(T_open_square)) { + int bound = 0; + + if (shape->array_dims >= 4) + error_at( + file_scope + ? "Array compound literal supports at most four dimensions" + : "Array declarators support at most four dimensions", + cur_token_loc()); + if (file_scope && lex_peek(T_close_square, NULL)) { + if (shape->array_dims) + error_at("Only the outer array bound may be omitted", + cur_token_loc()); + shape->array_outer_unsized = true; + lex_expect(T_close_square); + shape->array_dims++; + continue; + } + if (file_scope || lex_peek(T_numeric, NULL)) { + bound = read_const_expr(scope); + if (bound <= 0) + error_at("Array compound literal needs a positive bound", + file_scope ? cur_token_loc() : next_token_loc()); + } + if (!bound && shape->array_dims) + error_at("Only the outer array bound may be omitted", + cur_token_loc()); + set_array_dimension(&shape->array_size, &shape->array_dim2, + &shape->array_dim3, &shape->array_dim4, + shape->array_dims, bound, !file_scope); + if (!shape->array_dims && !bound) + shape->array_outer_unsized = true; + shape->array_dims++; + lex_expect(T_close_square); + } + return shape->array_dims != 0; +} + /* A subscript applied to the value on top of the operand stack, which has no * declaration for read_lvalue() to follow. */ @@ -332,6 +218,7 @@ static void lower_value_subscript(block_t *parent, basic_block_t **bb) int subscript_depth = 0; int array_dims = 1 + !!base->array_dim2 + !!base->array_dim3 + !!base->array_dim4; + fixed_array_shape_t shape = fixed_array_shape_from_var(base); func_t *element_signature = NULL; /* A row of callbacks, `(*rows)[i]` for a pointer to `fn_t[2]`, holds @@ -355,51 +242,17 @@ static void lower_value_subscript(block_t *parent, basic_block_t **bb) */ address = base; do { - int stride = element_size; - - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - index = opstack_pop(); - lex_expect(T_close_square); - if (subscript_depth == 0 && base->array_dim2) { - stride *= base->array_dim2; - if (base->array_dim3) - stride *= base->array_dim3; - if (base->array_dim4) - stride *= base->array_dim4; - } else if (subscript_depth == 1 && base->array_dim3) { - stride *= base->array_dim3; - if (base->array_dim4) - stride *= base->array_dim4; - } else if (subscript_depth == 2 && base->array_dim4) { - stride *= base->array_dim4; - } - if (stride != 1) { - var_t *scale = require_var(parent); - var_t *scaled = require_var(parent); - - scale->var_name = gen_name(); - scale->init_val = stride; - add_insn(parent, *bb, OP_load_constant, scale, NULL, NULL, 0, - NULL); - scaled->var_name = gen_name(); - add_insn(parent, *bb, OP_mul, scaled, index, scale, 0, NULL); - index = scaled; - } - var_t *indexed = - require_typed_ptr_var(parent, base->type, base->ptr_level + 1); - indexed->var_name = gen_name(); - add_insn(parent, *bb, OP_add, indexed, address, index, 0, NULL); - address = indexed; + address = read_subscript_address( + parent, bb, address, base->type, base->ptr_level + 1, + fixed_array_shape_stride(&shape, subscript_depth, + element_size)); subscript_depth++; } while (subscript_depth < array_dims && lex_accept(T_open_square)); if (subscript_depth < array_dims && !element_signature) { - opstack_push(array_subscript_remainder( - parent, bb, base->type, base->ptr_level, - fixed_array_shape_from_var(base), subscript_depth, address)); + opstack_push(array_subscript_remainder(parent, bb, base->type, + base->ptr_level, shape, + subscript_depth, address)); } else if (subscript_depth < array_dims) { opstack_push(address); } else { @@ -415,10 +268,12 @@ static void lower_value_subscript(block_t *parent, basic_block_t **bb) element_ptr_level += element_type->array_element_ptr_level; element_type = element_type->array_element_type; } - vd = require_typed_ptr_var(parent, element_type, element_ptr_level); - vd->var_name = gen_name(); + vd = name_var( + require_typed_ptr_var(parent, element_type, element_ptr_level)); vd->is_const_qualified = base->is_const_qualified; vd->pointer_const_mask = base->pointer_const_mask; + vd->pointer_volatile_mask = base->pointer_volatile_mask; + address->is_volatile_access = var_volatile_pointee(base); vd->is_const_pointer = base->is_const_pointer; if (element_signature) { vd->func_signature = element_signature; @@ -436,10 +291,7 @@ static void lower_value_subscript(block_t *parent, basic_block_t **bb) /* E1[E2] is (*((E1)+(E2))): pointer arithmetic scales the index by the * element size, and the dereference yields the element object. */ - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } + read_assignment_or_expression(parent, bb); index = opstack_pop(); lex_expect(T_close_square); if (is_pointer_like_value(base) == is_pointer_like_value(index) || @@ -505,8 +357,7 @@ static void lower_value_member(block_t *parent, basic_block_t **bb) * as a call or assignment result, is not. */ is_lvalue = base->is_compound_literal || is_named_object(base, parent); - address = require_ref_var(parent, base->type, 0); - address->var_name = gen_name(); + address = name_var(require_ref_var(parent, base->type, 0)); address->is_const_qualified = base->is_const_qualified; add_insn(parent, *bb, OP_address_of, address, base, NULL, 0, NULL); } @@ -525,7 +376,10 @@ void lower_postfix_operators(block_t *parent, basic_block_t **bb) var_t *top = operand_stack[operand_stack_idx - 1]; if (lex_peek(T_open_square, NULL)) { - if (top->pointee_array_size) + if (top->pointee_array_size && + (is_pointee_array_pointer(top) || + effective_pointer_depth(top) <= + top->pointee_array_element_ptr_level + 1)) lower_call_result_array_postfix(&top, parent, bb); else lower_value_subscript(parent, bb); @@ -561,18 +415,13 @@ void lower_postfix_operators(block_t *parent, basic_block_t **bb) if (lex_peek(T_increment, NULL) || lex_peek(T_decrement, NULL)) { var_t *object = opstack_pop(); - opcode_t op = OP_sub; - - if (lex_accept(T_increment)) - op = OP_add; - else - lex_expect(T_decrement); + opcode_t op = read_increment_operator(); if (!object->is_compound_literal_reference && is_named_object(object, parent)) { /* A grouped named object, `(count)++`, updates the variable. */ - var_t *old = - require_typed_ptr_var(parent, object->type, object->ptr_level); - var_t *one = require_typed_var(parent, TY_int); + var_t *old = name_var( + require_typed_ptr_var(parent, object->type, object->ptr_level)); + var_t *one; var_t *updated; if (object->array_size) @@ -584,27 +433,16 @@ void lower_postfix_operators(block_t *parent, basic_block_t **bb) "Increment or decrement requires a scalar modifiable " "lvalue", cur_token_loc()); - old->var_name = gen_name(); add_insn(parent, *bb, OP_assign, old, object, NULL, 0, NULL); - one->var_name = gen_name(); - one->init_val = 1; - add_insn(parent, *bb, OP_load_constant, one, NULL, NULL, 0, NULL); - if (is_pointer_operation(op, object, one)) { - handle_pointer_arithmetic(parent, bb, op, object, one); - updated = opstack_pop(); - } else { - updated = require_var(parent); - updated->var_name = gen_name(); - updated->type = integer_binary_result_type(op, object, one); - add_insn(parent, *bb, op, updated, object, one, 0, NULL); - } + one = load_constant(parent, *bb, 1, TY_int); + updated = combine_assignment_value(parent, bb, op, object, one); updated = resize_var(parent, bb, updated, object); mark_var_mutated(object); add_insn(parent, *bb, OP_assign, object, updated, NULL, 0, NULL); opstack_push(old); return; } - lower_reference_update(object, op, parent, bb); + lower_reference_update(object, op, parent, bb, false); /* The expression has the old value and is not itself an lvalue. */ object->is_compound_literal_reference = false; @@ -612,34 +450,53 @@ void lower_postfix_operators(block_t *parent, basic_block_t **bb) } } -/* A parenthesized expression, possibly a comma expression, and the postfix - * operators applied to its value. +/* Parse a full comma expression. Grouped operands clear compound-literal + * reference state on each right-hand value because the comma result is an + * ordinary expression value. */ -static void read_grouped_operand(block_t *parent, basic_block_t **bb) +static void read_comma_expression(block_t *parent, + basic_block_t **bb, + bool clear_reference) { - /* Regular parenthesized expression */ - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } + read_assignment_or_expression(parent, bb); while (lex_accept(T_comma)) { - /* A comma inside an explicit parenthesized expression is the C99 - * sequencing operator, not an argument or initializer delimiter. - * Discard the completed left value after its IR is emitted; the final - * expression supplies the result, which is not an lvalue. - */ discard_operand(parent, *bb); - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - operand_stack[operand_stack_idx - 1]->is_compound_literal_reference = - false; + read_assignment_or_expression(parent, bb); + if (clear_reference) + operand_stack[operand_stack_idx - 1] + ->is_compound_literal_reference = false; } +} + +/* A parenthesized expression, possibly a comma expression, and the postfix + * operators applied to its value. + */ +static void read_grouped_operand(block_t *parent, basic_block_t **bb) +{ + read_comma_expression(parent, bb, true); lex_expect(T_close_bracket); lower_postfix_operators(parent, bb); } +static void lower_compound_literal_assignment(block_t *parent, + basic_block_t **bb, + var_t *address, + bool is_const, + const lvalue_t *target) +{ + opcode_t op = OP_generic; + var_t *value; + + if (!accept_assignment_op(&op)) + return; + if (is_const) + error_at("assignment of read-only location", cur_token_loc()); + read_assignment_or_expression(parent, bb); + value = combine_addressed_assignment(parent, bb, op, address, target, + opstack_pop()); + store_addressed_assignment(parent, bb, address, target, value, false); +} + /* Subscripts, and the updates or stores that follow them, applied directly to * an array compound literal. */ @@ -653,44 +510,12 @@ static void lower_array_literal_subscripts(block_t *parent, var_t *element; int element_size = compound_var->type->size; int subscript_depth = 0; + fixed_array_shape_t shape = fixed_array_shape_from_var(compound_var); while (lex_accept(T_open_square)) { - var_t *index; - int stride = element_size; - - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - index = opstack_pop(); - lex_expect(T_close_square); - if (subscript_depth == 0 && compound_var->array_dim2) { - stride *= compound_var->array_dim2; - if (compound_var->array_dim3) - stride *= compound_var->array_dim3; - if (compound_var->array_dim4) - stride *= compound_var->array_dim4; - } else if (subscript_depth == 1 && compound_var->array_dim3) { - stride *= compound_var->array_dim3; - if (compound_var->array_dim4) - stride *= compound_var->array_dim4; - } else if (subscript_depth == 2 && compound_var->array_dim4) { - stride *= compound_var->array_dim4; - } - if (stride != 1) { - var_t *scale = require_var(parent); - scale->var_name = gen_name(); - scale->init_val = stride; - add_insn(parent, *bb, OP_load_constant, scale, NULL, NULL, 0, NULL); - var_t *scaled = require_var(parent); - scaled->var_name = gen_name(); - add_insn(parent, *bb, OP_mul, scaled, index, scale, 0, NULL); - index = scaled; - } - var_t *indexed = require_typed_ptr_var(parent, compound_var->type, 1); - indexed->var_name = gen_name(); - add_insn(parent, *bb, OP_add, indexed, address, index, 0, NULL); - address = indexed; + address = read_subscript_address( + parent, bb, address, compound_var->type, 1, + fixed_array_shape_stride(&shape, subscript_depth, element_size)); subscript_depth++; } @@ -699,88 +524,26 @@ static void lower_array_literal_subscripts(block_t *parent, * pointer value, not a scalar read, and keeps the bounds the subscripts * left for sizeof and for its own later subscripts. */ - opstack_push(array_subscript_remainder( - parent, bb, compound_var->type, compound_var->ptr_level, - fixed_array_shape_from_var(compound_var), subscript_depth, - address)); + opstack_push(array_subscript_remainder(parent, bb, compound_var->type, + compound_var->ptr_level, shape, + subscript_depth, address)); } else { - opcode_t compound_op = OP_generic; - if (lex_accept(T_assign) || accept_compound_assign_op(&compound_op)) { - if (compound_var->is_const_qualified) - error_at("assignment of read-only location", cur_token_loc()); - var_t *value; - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - value = opstack_pop(); - if (compound_op != OP_generic) { - var_t *current = require_typed_var(parent, compound_var->type); - current->var_name = gen_name(); - add_insn(parent, *bb, OP_read, current, address, NULL, - element_size, NULL); - /* A pointer object steps by its element size. */ - if (is_pointer_operation(compound_op, current, value)) { - handle_pointer_arithmetic(parent, bb, compound_op, current, - value); - value = opstack_pop(); - } else { - current = integer_promote_operand(parent, bb, current); - value = integer_promote_operand(parent, bb, value); - normalize_integer_binary_operands(parent, bb, compound_op, - ¤t, &value); - var_t *combined = require_var(parent); - combined->var_name = gen_name(); - combined->type = - integer_binary_result_type(compound_op, current, value); - add_insn(parent, *bb, compound_op, combined, current, value, - 0, NULL); - value = combined; - } - } - value = convert_stored_value(parent, bb, value, compound_var->type, - compound_var->ptr_level); - add_insn(parent, *bb, OP_write, NULL, address, value, element_size, - NULL); - } + lower_compound_literal_assignment( + parent, bb, address, compound_var->is_const_qualified, + &(lvalue_t) {.size = element_size, + .value_ptr_level = compound_var->ptr_level, + .type = compound_var->type}); + element = name_var(require_typed_var(parent, compound_var->type)); + add_insn(parent, *bb, OP_read, element, address, NULL, element_size, + NULL); + element->is_const_qualified = compound_var->is_const_qualified; + mark_value_reference(element, address, NULL); if (lex_peek(T_increment, NULL) || lex_peek(T_decrement, NULL)) { - opcode_t op = OP_sub; - var_t *one; - var_t *updated; - - if (lex_accept(T_increment)) - op = OP_add; - else - lex_expect(T_decrement); - - if (compound_var->is_const_qualified) - error_at("assignment of read-only location", cur_token_loc()); - element = require_typed_var(parent, compound_var->type); - element->var_name = gen_name(); - add_insn(parent, *bb, OP_read, element, address, NULL, element_size, - NULL); - one = require_typed_var(parent, TY_int); - one->var_name = gen_name(); - one->init_val = 1; - add_insn(parent, *bb, OP_load_constant, one, NULL, NULL, 0, NULL); - updated = require_var(parent); - updated->var_name = gen_name(); - updated->type = integer_binary_result_type(op, element, one); - add_insn(parent, *bb, op, updated, element, one, 0, NULL); - updated = resize_var(parent, bb, updated, element); - add_insn(parent, *bb, OP_write, NULL, address, updated, - element_size, NULL); - opstack_push(element); - } else { - element = require_typed_var(parent, compound_var->type); - element->var_name = gen_name(); - add_insn(parent, *bb, OP_read, element, address, NULL, element_size, - NULL); - element->is_compound_literal_reference = true; - element->is_const_qualified = compound_var->is_const_qualified; - element->compound_literal_address = address; - opstack_push(element); + lower_reference_update(element, read_increment_operator(), parent, + bb, false); + element->is_compound_literal_reference = false; } + opstack_push(element); } } @@ -794,17 +557,15 @@ static void lower_record_literal_members(block_t *parent, { char field_name[MAX_ID_LEN]; var_t *base = opstack_pop(); - var_t *base_addr = require_ref_var(parent, base->type, 0); + var_t *base_addr = name_var(require_ref_var(parent, base->type, 0)); var_t *field; var_t *address; var_t *element; - opcode_t compound_op = OP_generic; type_t *value_type; int value_ptr_level; int value_size; type_t *record_type = tn->type; - base_addr->var_name = gen_name(); add_insn(parent, *bb, OP_address_of, base_addr, base, NULL, 0, NULL); address = base_addr; while (1) { @@ -836,135 +597,44 @@ static void lower_record_literal_members(block_t *parent, * remaining dimensions, just as ordinary array postfix parsing does. */ int subscript_depth = 0; + fixed_array_shape_t shape = fixed_array_shape_from_var(field); while (field->array_size && lex_accept(T_open_square)) { - var_t *index; - int stride = value_size; - - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - index = opstack_pop(); - lex_expect(T_close_square); - if (subscript_depth == 0 && field->array_dim2 > 0) { - stride *= field->array_dim2; - if (field->array_dim3 > 0) - stride *= field->array_dim3; - if (field->array_dim4 > 0) - stride *= field->array_dim4; - } else if (subscript_depth == 1 && field->array_dim3 > 0) { - stride *= field->array_dim3; - if (field->array_dim4 > 0) - stride *= field->array_dim4; - } else if (subscript_depth == 2 && field->array_dim4 > 0) { - stride *= field->array_dim4; - } - if (stride != 1) { - var_t *scale = require_var(parent); - scale->var_name = gen_name(); - scale->init_val = stride; - add_insn(parent, *bb, OP_load_constant, scale, NULL, NULL, 0, NULL); - var_t *scaled = require_var(parent); - scaled->var_name = gen_name(); - add_insn(parent, *bb, OP_mul, scaled, index, scale, 0, NULL); - index = scaled; - } - var_t *indexed = require_typed_ptr_var(parent, value_type, 1); - indexed->var_name = gen_name(); - add_insn(parent, *bb, OP_add, indexed, address, index, 0, NULL); - address = indexed; + address = read_subscript_address( + parent, bb, address, value_type, 1, + fixed_array_shape_stride(&shape, subscript_depth, value_size)); value_ptr_level = 0; subscript_depth++; } - if (lex_accept(T_assign) || accept_compound_assign_op(&compound_op)) { - if (compound_object->is_const_qualified) - error_at("assignment of read-only location", cur_token_loc()); - var_t *value; - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - value = opstack_pop(); - if (compound_op != OP_generic) { - var_t *current; - if (is_bitfield(field)) - current = read_bitfield_value(parent, bb, address, field); - else { - current = require_typed_var(parent, value_type); - current->ptr_level = value_ptr_level; - current->var_name = gen_name(); - add_insn(parent, *bb, OP_read, current, address, NULL, - value_size, NULL); - } - /* A pointer object steps by its element size. */ - if (is_pointer_operation(compound_op, current, value)) { - handle_pointer_arithmetic(parent, bb, compound_op, current, - value); - value = opstack_pop(); - } else { - current = integer_promote_operand(parent, bb, current); - value = integer_promote_operand(parent, bb, value); - normalize_integer_binary_operands(parent, bb, compound_op, - ¤t, &value); - var_t *combined = require_var(parent); - combined->var_name = gen_name(); - combined->type = - integer_binary_result_type(compound_op, current, value); - add_insn(parent, *bb, compound_op, combined, current, value, 0, - NULL); - value = combined; - } - } - if (is_bitfield(field)) - write_bitfield_value(parent, bb, address, value, field); - else { - value = convert_stored_value(parent, bb, value, value_type, - value_ptr_level); - add_insn(parent, *bb, OP_write, NULL, address, value, value_size, - NULL); - } - } + lower_compound_literal_assignment( + parent, bb, address, compound_object->is_const_qualified, + &(lvalue_t) {.size = value_size, + .value_ptr_level = value_ptr_level, + .type = value_type, + .decl = field}); if (lex_peek(T_increment, NULL) || lex_peek(T_decrement, NULL)) { - opcode_t op = lex_accept(T_increment) ? OP_add : OP_sub; + opcode_t op = read_increment_operator(); var_t *old; - var_t *one; - var_t *updated; - if (compound_object->is_const_qualified) - error_at("assignment of read-only location", cur_token_loc()); if (is_bitfield(field)) old = read_bitfield_value(parent, bb, address, field); else { - old = require_typed_var(parent, value_type); - old->ptr_level = value_ptr_level; - old->var_name = gen_name(); + old = name_var( + require_typed_ptr_var(parent, value_type, value_ptr_level)); add_insn(parent, *bb, OP_read, old, address, NULL, value_size, NULL); } - one = require_typed_var(parent, TY_int); - one->var_name = gen_name(); - one->init_val = 1; - add_insn(parent, *bb, OP_load_constant, one, NULL, NULL, 0, NULL); - updated = require_var(parent); - updated->var_name = gen_name(); - updated->type = integer_binary_result_type(op, old, one); - add_insn(parent, *bb, op, updated, old, one, 0, NULL); - updated = resize_var(parent, bb, updated, old); - if (is_bitfield(field)) - write_bitfield_value(parent, bb, address, updated, field); - else - add_insn(parent, *bb, OP_write, NULL, address, updated, value_size, - NULL); + old->is_const_qualified = compound_object->is_const_qualified; + mark_value_reference(old, address, is_bitfield(field) ? field : NULL); + lower_reference_update(old, op, parent, bb, false); element = old; } else if (is_bitfield(field)) { element = read_bitfield_value(parent, bb, address, field); element->is_bitfield = false; } else { - element = require_typed_var(parent, value_type); - element->ptr_level = value_ptr_level; - element->var_name = gen_name(); + element = name_var( + require_typed_ptr_var(parent, value_type, value_ptr_level)); add_insn(parent, *bb, OP_read, element, address, NULL, value_size, NULL); } @@ -982,11 +652,9 @@ static void lower_record_literal_members(block_t *parent, if (field->array_dim4) field_dims++; if (!field->array_size || subscript_depth >= field_dims) { - element->is_compound_literal_reference = true; element->is_const_qualified = compound_object->is_const_qualified; - element->compound_literal_address = address; - if (is_bitfield(field)) - element->compound_literal_bitfield = field; + mark_value_reference(element, address, + is_bitfield(field) ? field : NULL); } opstack_push(element); } @@ -1002,8 +670,7 @@ static void read_compound_literal_operand(block_t *parent, var_t *compound_object = NULL; /* Create variable for compound literal result */ - var_t *compound_var = require_typed_var(parent, tn->type); - compound_var->var_name = gen_name(); + var_t *compound_var = name_var(require_typed_var(parent, tn->type)); compound_var->is_compound_literal = true; compound_var->is_const_qualified = tn->const_qualified; compound_var->is_const_pointer = tn->const_pointer; @@ -1022,14 +689,10 @@ static void read_compound_literal_operand(block_t *parent, tn->ptr_level = 0; /* Reset for normal processing */ bool consumed_close_brace = false; - /* parse_array_init() consumes its own opening brace. The older - * one-dimensional literal helper expects it already consumed. - */ - if (!is_array_literal || - (tn->array_dims <= 1 && !tn->array_element_ptr_level)) + /* Array initialization owns its opening brace. */ + if (!is_array_literal) lex_expect(T_open_curly); - if (!is_array_literal || - (tn->array_dims <= 1 && !tn->array_element_ptr_level)) + if (!is_array_literal) reject_empty_initializer_in_strict_c99(); /* Check if this is a pointer compound literal */ if (is_array_literal) { @@ -1044,10 +707,7 @@ static void read_compound_literal_operand(block_t *parent, compound_var->pointee_array_dim3 = tn->pointee_array_dim3; compound_var->pointee_array_dim4 = tn->pointee_array_dim4; add_insn(parent, *bb, OP_allocat, compound_var, NULL, NULL, 0, NULL); - if (tn->array_dims > 1 || tn->array_element_ptr_level) - parse_array_init(compound_var, parent, bb); - else - parse_array_compound_literal(compound_var, parent, bb); + parse_array_init(compound_var, parent, bb); if (compound_var->array_size == 0) { compound_var->init_val = 0; @@ -1077,8 +737,7 @@ static void read_compound_literal_operand(block_t *parent, compound_var->pointee_func_signature = tn->func_signature; } if (lex_peek(T_close_curly, NULL)) { - initializer = require_var(parent); - initializer->var_name = gen_name(); + initializer = require_named_var(parent); initializer->is_const = true; add_insn(parent, *bb, OP_load_constant, initializer, NULL, NULL, 0, NULL); @@ -1121,9 +780,8 @@ static void read_compound_literal_operand(block_t *parent, false); } else { /* Empty pointer compound literal: (int*){} */ - initializer = require_typed_var(parent, tn->type); - initializer->ptr_level = tn->ptr_level; - initializer->var_name = gen_name(); + initializer = name_var( + require_typed_ptr_var(parent, tn->type, tn->ptr_level)); initializer->init_val = 0; add_insn(parent, *bb, OP_load_constant, initializer, NULL, NULL, 0, NULL); @@ -1147,8 +805,8 @@ static void read_compound_literal_operand(block_t *parent, struct_type = struct_type->base_struct; add_insn(parent, *bb, OP_allocat, compound_var, NULL, NULL, 0, NULL); - var_t *compound_addr = require_ref_var(parent, compound_var->type, 0); - compound_addr->var_name = gen_name(); + var_t *compound_addr = + name_var(require_ref_var(parent, compound_var->type, 0)); add_insn(parent, *bb, OP_address_of, compound_addr, compound_var, NULL, 0, NULL); parse_struct_field_init(parent, bb, struct_type, compound_addr); @@ -1166,10 +824,7 @@ static void read_compound_literal_operand(block_t *parent, compound_var->array_size = 0; add_insn(parent, *bb, OP_allocat, compound_var, NULL, NULL, 0, NULL); - var_t *zero = require_typed_var(parent, tn->type); - zero->var_name = gen_name(); - zero->init_val = 0; - add_insn(parent, *bb, OP_load_constant, zero, NULL, NULL, 0, NULL); + var_t *zero = load_constant(parent, *bb, 0, tn->type); emit_object_assignment(parent, bb, compound_var, zero); opstack_push(compound_var); compound_object = compound_var; @@ -1185,94 +840,15 @@ static void read_compound_literal_operand(block_t *parent, error_at("Too many elements in scalar compound literal", cur_token_loc()); - /* Retained for the parser's array-literal extension; scalar - * spellings above have already rejected a second initializer. - */ - if (lex_peek(T_comma, NULL)) { - /* Array compound literal: (int[]){1, 2, 3} */ - var_t *first_element = opstack_pop(); - - /* Store elements temporarily */ - var_t *elements[256]; - elements[0] = first_element; - int element_count = 1; - - /* Parse remaining elements */ - while (lex_accept(T_comma)) { - if (lex_peek(T_close_curly, NULL)) - break; /* Trailing comma */ - - read_expr(parent, bb); - read_ternary_operation(parent, bb); - if (element_count < 256) { - elements[element_count] = opstack_pop(); - } else { - opstack_pop(); /* Discard if too many */ - } - element_count++; - } - - /* Set array metadata */ - compound_var->array_size = element_count; - compound_var->init_val = first_element->init_val; - - /* Allocate space for the array on stack */ - add_insn(parent, *bb, OP_allocat, compound_var, NULL, NULL, 0, - NULL); - - /* Initialize each element */ - for (int i = 0; i < element_count && i < 256; i++) { - if (!elements[i]) - continue; - - /* Store element at offset i * sizeof(element) */ - var_t *elem_offset = require_var(parent); - elem_offset->init_val = i * tn->type->size; - elem_offset->var_name = gen_name(); - add_insn(parent, *bb, OP_load_constant, elem_offset, NULL, - NULL, 0, NULL); - - /* Calculate address of element */ - var_t *elem_addr = require_var(parent); - elem_addr->ptr_level = 1; - elem_addr->var_name = gen_name(); - add_insn(parent, *bb, OP_add, elem_addr, compound_var, - elem_offset, 0, NULL); - - /* Store the element value */ - add_insn(parent, *bb, OP_write, NULL, elem_addr, - elements[i], tn->type->size, NULL); - } - - /* Store first element value for array-to-scalar */ - compound_var->init_val = first_element->init_val; - - /* Create result that provides first element access. This - * enables array compound literals in scalar contexts: int x = - * (int[]){1,2,3}; // x gets 1 int y = 5 + (int[]){10}; // adds - * 5 + 10 - */ - var_t *result_var = require_var(parent); - result_var->var_name = gen_name(); - result_var->type = compound_var->type; - result_var->ptr_level = 0; - result_var->array_size = 0; - - /* Read first element from the array */ - add_insn(parent, *bb, OP_read, result_var, compound_var, NULL, - compound_var->type->size, NULL); - opstack_push(result_var); - } else { - /* Single value: (int){42} - scalar compound literal */ - var_t *initializer = opstack_pop(); - diagnose_integer_to_pointer_conversion(initializer, - compound_var, false); - add_insn(parent, *bb, OP_allocat, compound_var, NULL, NULL, 0, - NULL); - emit_object_assignment(parent, bb, compound_var, initializer); - compound_object = compound_var; - opstack_push(compound_var); - } + /* Single value: (int){42} - scalar compound literal */ + var_t *initializer = opstack_pop(); + diagnose_integer_to_pointer_conversion(initializer, compound_var, + false); + add_insn(parent, *bb, OP_allocat, compound_var, NULL, NULL, 0, + NULL); + emit_object_assignment(parent, bb, compound_var, initializer); + compound_object = compound_var; + opstack_push(compound_var); } } @@ -1308,7 +884,7 @@ static void read_compound_literal_operand(block_t *parent, */ if (compound_object && !is_record_type(tn->type) && (lex_peek(T_increment, NULL) || lex_peek(T_decrement, NULL))) { - opcode_t op = lex_accept(T_increment) ? OP_add : OP_sub; + opcode_t op = read_increment_operator(); var_t *old; var_t *one; var_t *updated; @@ -1317,62 +893,28 @@ static void read_compound_literal_operand(block_t *parent, compound_object->is_const_pointer) error_at("assignment of read-only location", cur_token_loc()); opstack_pop(); - old = require_typed_var(parent, compound_object->type); - old->ptr_level = compound_object->ptr_level; - old->var_name = gen_name(); + old = name_var(require_typed_ptr_var(parent, compound_object->type, + compound_object->ptr_level)); add_insn(parent, *bb, OP_assign, old, compound_object, NULL, 0, NULL); - one = require_typed_var(parent, TY_int); - one->var_name = gen_name(); - one->init_val = 1; - add_insn(parent, *bb, OP_load_constant, one, NULL, NULL, 0, NULL); - if (is_pointer_operation(op, compound_object, one)) { - handle_pointer_arithmetic(parent, bb, op, compound_object, one); - updated = opstack_pop(); - } else { - updated = require_var(parent); - updated->var_name = gen_name(); - updated->type = - integer_binary_result_type(op, compound_object, one); - add_insn(parent, *bb, op, updated, compound_object, one, 0, NULL); - } + one = load_constant(parent, *bb, 1, TY_int); + updated = + combine_assignment_value(parent, bb, op, compound_object, one); updated = resize_var(parent, bb, updated, compound_object); add_insn(parent, *bb, OP_assign, compound_object, updated, NULL, 0, NULL); opstack_push(old); } opcode_t scalar_compound_op = OP_generic; - if (compound_object && (lex_accept(T_assign) || - accept_compound_assign_op(&scalar_compound_op))) { + if (compound_object && accept_assignment_op(&scalar_compound_op)) { if (compound_object->is_const_qualified || compound_object->is_const_pointer) error_at("assignment of read-only location", cur_token_loc()); opstack_pop(); - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } + read_assignment_or_expression(parent, bb); var_t *value = opstack_pop(); - if (scalar_compound_op != OP_generic) { - var_t *current = compound_object; - - if (is_pointer_operation(scalar_compound_op, current, value)) { - handle_pointer_arithmetic(parent, bb, scalar_compound_op, - current, value); - value = opstack_pop(); - } else { - current = integer_promote_operand(parent, bb, current); - value = integer_promote_operand(parent, bb, value); - normalize_integer_binary_operands( - parent, bb, scalar_compound_op, ¤t, &value); - var_t *combined = require_var(parent); - combined->var_name = gen_name(); - combined->type = integer_binary_result_type(scalar_compound_op, - current, value); - add_insn(parent, *bb, scalar_compound_op, combined, current, - value, 0, NULL); - value = combined; - } - } + if (scalar_compound_op != OP_generic) + value = combine_assignment_value(parent, bb, scalar_compound_op, + compound_object, value); emit_object_assignment(parent, bb, compound_object, value); opstack_push(compound_object); } @@ -1385,6 +927,19 @@ static void read_compound_literal_operand(block_t *parent, * source payload: `(unsigned char)256`, `(short)65536`, and a 64-to-32 cast all * become the integer constant zero and may therefore serve as null pointers. */ +static int sign_extend_narrow_int(unsigned int value, int size) +{ + unsigned int bits, mask, sign; + + if (size != 1 && size != 2) + return (int) value; + bits = size * 8; + mask = (1U << bits) - 1; + sign = 1U << (bits - 1); + value &= mask; + return value & sign ? (int) value - (int) (sign << 1) : (int) value; +} + void fold_integer_constant_cast(var_t *result, unsigned int lo, unsigned int hi) { if (result->type->is_bool) { @@ -1398,9 +953,9 @@ void fold_integer_constant_cast(var_t *result, unsigned int lo, unsigned int hi) unsigned int bits = result->type->size * 8; unsigned int mask = (1U << bits) - 1; - lo &= mask; - if (!result->type->is_unsigned && (lo & (1U << (bits - 1)))) - lo |= ~mask; + lo = result->type->is_unsigned ? lo & mask + : (unsigned int) sign_extend_narrow_int( + lo, result->type->size); result->init_val = (int) lo; result->init_val_hi = result->init_val < 0 ? -1 : 0; } else if (result->type->size == TY_int->size) { @@ -1443,8 +998,8 @@ static void read_cast_operand(block_t *parent, expr_var = materialize_function_designator(parent, bb, expr_var); /* Create variable for cast result */ - var_t *cast_var = require_typed_ptr_var(parent, tn->type, tn->ptr_level); - cast_var->var_name = gen_name(); + var_t *cast_var = + name_var(require_typed_ptr_var(parent, tn->type, tn->ptr_level)); cast_var->is_const_qualified = tn->const_qualified; cast_var->is_const_pointer = tn->const_pointer; cast_var->pointer_const_mask = tn->pointer_const_mask; @@ -1627,6 +1182,9 @@ static void read_parenthesized_operand(block_t *parent, basic_block_t **bb) /* Check for pointer types: int*, char*, etc. */ int ptr_level = 0; + var_t return_decl; + init_type_name_decl(&return_decl, type, has_const_type, + tn.volatile_qualified); while (lex_accept(T_asterisk)) { ptr_level++; while (lex_peek(T_const, NULL) || lex_peek(T_volatile, NULL) || @@ -1635,9 +1193,14 @@ static void read_parenthesized_operand(block_t *parent, basic_block_t **bb) tn.const_pointer = true; if (ptr_level <= 32) tn.pointer_const_mask |= 1U << (ptr_level - 1); - } else if (lex_accept(T_volatile)) + if (ptr_level <= 32) + return_decl.pointer_const_mask |= 1U << (ptr_level - 1); + } else if (lex_accept(T_volatile)) { + int depth = ptr_level + type->ptr_level; + if (depth <= 32) + return_decl.pointer_volatile_mask |= 1U << (depth - 1); tn.volatile_qualified = true; - else + } else lex_expect(T_restrict); } } @@ -1645,115 +1208,13 @@ static void read_parenthesized_operand(block_t *parent, basic_block_t **bb) bool is_array = false; if (abstract_function_pointer_follows()) { + return_decl.ptr_level = ptr_level; tn.func_signature = read_abstract_function_pointer( - type, ptr_level, &tn.func_pointer_level); + &return_decl, &tn.func_pointer_level); ptr_level = 0; } - /* A parenthesized abstract declarator, such as `(int (*[])[2]){...}`, - * is an array whose elements are pointers to rows. Keep the outer array - * and pointee bounds separate, matching named pointer-to-array - * declarations. - */ - if (lex_accept(T_open_bracket)) { - int pointee_dims = 0; - - if (ptr_level || !lex_peek(T_asterisk, NULL)) - error_at("Array compound literal needs a pointer declarator", - cur_token_loc()); - do { - lex_expect(T_asterisk); - tn.array_element_ptr_level++; - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; - } while (lex_peek(T_asterisk, NULL)); - lex_expect(T_open_square); - if (!lex_peek(T_close_square, NULL)) { - tn.array_size = read_const_expr(parent); - if (tn.array_size <= 0) - error_at("Array compound literal needs a positive bound", - cur_token_loc()); - } else { - tn.array_outer_unsized = true; - } - lex_expect(T_close_square); - lex_expect(T_close_bracket); - while (lex_accept(T_open_square)) { - int bound = read_const_expr(parent); - - if (pointee_dims >= 4) - error_at( - "Array declarators support at most four " - "dimensions", - cur_token_loc()); - if (bound <= 0) - error_at("Array size must be positive", cur_token_loc()); - if (pointee_dims == 0) - tn.pointee_array_size = bound; - else { - if (pointee_dims == 1) - tn.pointee_array_dim2 = bound; - else if (pointee_dims == 2) - tn.pointee_array_dim3 = bound; - else - tn.pointee_array_dim4 = bound; - tn.pointee_array_size *= bound; - } - lex_expect(T_close_square); - pointee_dims++; - } - if (!pointee_dims) - error_at("Array compound literal needs a row bound", - cur_token_loc()); - is_array = true; - tn.array_dims = 1; - tn.parenthesized_array = true; - } - - /* Parse every array bound in a compound-literal type name. `var_t` - * records the complete flattened element count plus up to three - * trailing dimensions, the same representation ordinary declarators use - * for `int a[2][3]`. - */ - while (!tn.parenthesized_array && lex_accept(T_open_square)) { - int bound = 0; - - is_array = true; - if (tn.array_dims >= 4) - error_at("Array declarators support at most four dimensions", - cur_token_loc()); - if (lex_peek(T_numeric, NULL)) { - /* The constant expression reader keeps a literal too wide for - * an int from wrapping into a small bound. - */ - bound = read_const_expr(parent); - if (bound <= 0) - error_at("Array compound literal needs a positive bound", - next_token_loc()); - } - if (!bound && tn.array_dims) - error_at("Only the outer array bound may be omitted", - cur_token_loc()); - if (!tn.array_dims) - tn.array_size = bound; - else { - if (tn.array_size) - tn.array_size *= bound; - else - tn.array_size = bound; - if (tn.array_dims == 1) - tn.array_dim2 = bound; - else if (tn.array_dims == 2) - tn.array_dim3 = bound; - else - tn.array_dim4 = bound; - } - if (!tn.array_dims && !bound) - tn.array_outer_unsized = true; - tn.array_dims++; - lex_expect(T_close_square); - } + is_array = read_compound_array_shape(parent, ptr_level, false, &tn); if (!is_array && type->array_size) { is_array = true; tn.array_size = type->array_size; @@ -1805,35 +1266,42 @@ static void read_parenthesized_operand(block_t *parent, basic_block_t **bb) } } -/* Replace the operand on top of the stack with its negation, the unary minus of - * C99 6.5.3.3, folding a constant. +/* Lower an integer unary operation, folding constants with the operand's type. */ -static void negate_operand(block_t *parent, basic_block_t **bb) +static void read_integer_unary_operand(block_t *parent, + basic_block_t **bb, + opcode_t op) { var_t *rs1 = opstack_pop(); var_t *vd; reject_record_operand(rs1); if (is_pointer_like_value(rs1) || rs1->is_func) - error_at("unary minus requires an arithmetic operand", cur_token_loc()); + error_at(op == OP_negate + ? "unary minus requires an arithmetic operand" + : "bitwise complement requires an integer operand", + cur_token_loc()); rs1 = integer_promote_operand(parent, bb, rs1); - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); vd->type = rs1->type; - opstack_push(vd); if (rs1->is_const && !rs1->ptr_level && !rs1->is_global) { vd->is_const = true; - vd->init_val = -rs1->init_val; - vd->init_val_hi = ~rs1->init_val_hi + (vd->init_val == 0); + vd->init_val = op == OP_negate ? -rs1->init_val : ~rs1->init_val; + vd->init_val_hi = op == OP_negate + ? ~rs1->init_val_hi + (vd->init_val == 0) + : ~rs1->init_val_hi; - /* The high word of an int result follows its type, as for a cast: "-9U" - * is 0xfffffff7 with a zero high word. + /* Preserve the promoted type: -9U has a zero high word, and + * ~(4294967294U) is 1 rather than a sign-extended 64-bit value. */ fold_integer_constant_cast(vd, vd->init_val, vd->init_val_hi); + opstack_push(vd); add_insn(parent, *bb, OP_load_constant, vd, NULL, NULL, 0, NULL); - } else - add_insn(parent, *bb, OP_negate, vd, rs1, NULL, 0, NULL); + } else { + opstack_push(vd); + add_insn(parent, *bb, op, vd, rs1, NULL, 0, NULL); + } } static void read_expr_operand_body(block_t *parent, basic_block_t **bb) @@ -1841,17 +1309,6 @@ static void read_expr_operand_body(block_t *parent, basic_block_t **bb) var_t *vd, *rs1; bool is_neg = false; - if (grouped_scalar_pointee_row_postfix_starts(parent)) { - handle_grouped_scalar_pointee_row_postfix(parent, bb); - return; - } - - if ((lex_peek(T_increment, NULL) || lex_peek(T_decrement, NULL)) && - grouped_scalar_pointee_row_prefix_starts(parent)) { - handle_grouped_scalar_pointee_row_prefix(parent, bb); - return; - } - bool prefix_increment = lex_peek(T_increment, NULL); bool prefix_decrement = lex_peek(T_decrement, NULL); if ((prefix_increment || prefix_decrement) && cur_token->next->next && @@ -1873,7 +1330,7 @@ static void read_expr_operand_body(block_t *parent, basic_block_t **bb) read_expr_operand(parent, bb); object = opstack_pop(); if (object->is_compound_literal_reference) { - opstack_push(lower_reference_update(object, op, parent, bb)); + opstack_push(lower_reference_update(object, op, parent, bb, true)); return; } if (object->array_size && is_named_object(object, parent)) @@ -1887,19 +1344,8 @@ static void read_expr_operand_body(block_t *parent, basic_block_t **bb) cur_token_loc()); if (object->is_const_qualified) error_at("assignment of read-only location", cur_token_loc()); - one = require_typed_var(parent, TY_int); - one->var_name = gen_name(); - one->init_val = 1; - add_insn(parent, *bb, OP_load_constant, one, NULL, NULL, 0, NULL); - if (is_pointer_operation(op, object, one)) { - handle_pointer_arithmetic(parent, bb, op, object, one); - vd = opstack_pop(); - } else { - vd = require_var(parent); - vd->var_name = gen_name(); - vd->type = integer_binary_result_type(op, object, one); - add_insn(parent, *bb, op, vd, object, one, 0, NULL); - } + one = load_constant(parent, *bb, 1, TY_int); + vd = combine_assignment_value(parent, bb, op, object, one); vd = resize_var(parent, bb, vd, object); mark_var_mutated(object); add_insn(parent, *bb, OP_assign, object, vd, NULL, 0, NULL); @@ -1933,7 +1379,7 @@ static void read_expr_operand_body(block_t *parent, basic_block_t **bb) (cur_token->next->next && cur_token->next->next->kind == T_open_square)) { read_expr_operand(parent, bb); - negate_operand(parent, bb); + read_integer_unary_operand(parent, bb, OP_negate); return; } is_neg = true; @@ -1978,16 +1424,14 @@ static void read_expr_operand_body(block_t *parent, basic_block_t **bb) /* Constant folding for logical NOT */ if (rs1 && rs1->is_const && !rs1->ptr_level && !rs1->is_global) { - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); vd->type = TY_int; vd->is_const = true; vd->init_val = !(rs1->init_val || rs1->init_val_hi); opstack_push(vd); add_insn(parent, *bb, OP_load_constant, vd, NULL, NULL, 0, NULL); } else { - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); /* C99 6.5.3.3: logical negation always yields int. Preserving the * operand's type made !pointer inherit the pointed-to record size, @@ -2001,35 +1445,7 @@ static void read_expr_operand_body(block_t *parent, basic_block_t **bb) } } else if (lex_accept(T_bit_not)) { read_expr_operand(parent, bb); - - rs1 = opstack_pop(); - reject_record_operand(rs1); - if (is_pointer_like_value(rs1) || rs1->is_func) - error_at("bitwise complement requires an integer operand", - cur_token_loc()); - rs1 = integer_promote_operand(parent, bb, rs1); - - /* Constant folding for bitwise NOT */ - if (rs1 && rs1->is_const && !rs1->ptr_level && !rs1->is_global) { - vd = require_var(parent); - vd->var_name = gen_name(); - vd->type = rs1->type; - vd->is_const = true; - vd->init_val = ~rs1->init_val; - vd->init_val_hi = ~rs1->init_val_hi; - - /* "(unsigned long long) ~4294967294U" is 1, not 0xffffffff00000001. - */ - fold_integer_constant_cast(vd, vd->init_val, vd->init_val_hi); - opstack_push(vd); - add_insn(parent, *bb, OP_load_constant, vd, NULL, NULL, 0, NULL); - } else { - vd = require_var(parent); - vd->var_name = gen_name(); - vd->type = rs1->type; - opstack_push(vd); - add_insn(parent, *bb, OP_bit_not, vd, rs1, NULL, 0, NULL); - } + read_integer_unary_operand(parent, bb, OP_bit_not); } else if (lex_accept(T_ampersand)) { handle_address_of_operator(parent, bb); } else if (lex_accept(T_asterisk)) { @@ -2069,6 +1485,8 @@ static void read_expr_operand_body(block_t *parent, basic_block_t **bb) if (!strcmp(token, "__builtin_offsetof")) { read_builtin_offsetof(parent, bb); + } else if (!strcmp(token, "__builtin_va_start")) { + read_builtin_va_start(parent, bb); } else if (!strcmp(token, "__builtin_va_arg")) { read_builtin_va_arg(parent, bb); } else if (!strcmp(token, "__func__")) { @@ -2076,27 +1494,16 @@ static void read_expr_operand_body(block_t *parent, basic_block_t **bb) error_at("__func__ is only defined inside a function", next_token_loc()); lex_expect(T_identifier); - vd = require_typed_ptr_var(parent, TY_char, true); - vd->var_name = gen_name(); - vd->init_val = write_symbol(parent->func->return_def.var_name); + vd = name_var(require_typed_ptr_var(parent, TY_char, true)); + vd->init_val = write_symbol(function_source_name(parent->func)); vd->is_string_literal = true; opstack_push(vd); add_insn(parent, *bb, OP_load_rodata_address, vd, NULL, NULL, 0, NULL); if (lex_accept(T_open_square)) { - var_t *base = opstack_pop(); - var_t *index; - var_t *address; - - read_expr(parent, bb); - read_ternary_operation(parent, bb); - index = opstack_pop(); - lex_expect(T_close_square); - address = require_typed_ptr_var(parent, TY_char, true); - address->var_name = gen_name(); - add_insn(parent, *bb, OP_add, address, base, index, 0, NULL); - vd = require_typed_var(parent, TY_char); - vd->var_name = gen_name(); + var_t *address = read_subscript_address( + parent, bb, opstack_pop(), TY_char, true, 1); + vd = name_var(require_typed_var(parent, TY_char)); opstack_push(vd); add_insn(parent, *bb, OP_read, vd, address, NULL, 1, NULL); } @@ -2104,10 +1511,9 @@ static void read_expr_operand_body(block_t *parent, basic_block_t **bb) /* An enumeration constant is an integer constant, so one that is * zero is a null pointer constant (C99 6.3.2.3p3). */ - vd = require_var(parent); + vd = name_var(require_var(parent)); vd->init_val = con->value; vd->is_const = true; - vd->var_name = gen_name(); opstack_push(vd); lex_expect(T_identifier); add_insn(parent, *bb, OP_load_constant, vd, NULL, NULL, 0, NULL); @@ -2193,9 +1599,7 @@ static void read_expr_operand_body(block_t *parent, basic_block_t **bb) lower_call_result_prefix_update(&vd, prefix_op, parent, bb); } else { /* indirective function pointer assignment */ - vd = require_func_symbol_var(parent); - vd->is_func = true; - vd->var_name = intern_string(token); + vd = function_designator_value(parent, func); opstack_push(vd); } } else if (lex_accept(T_open_curly)) { @@ -2233,6 +1637,46 @@ bool is_logical(opcode_t op) return op == OP_log_and || op == OP_log_or; } +static opcode_t compound_assignment_opcode(token_kind_t kind) +{ + switch (kind) { + case T_pluseq: + return OP_add; + case T_minuseq: + return OP_sub; + case T_asteriskeq: + return OP_mul; + case T_divideeq: + return OP_div; + case T_modeq: + return OP_mod; + case T_lshifteq: + return OP_lshift; + case T_rshifteq: + return OP_rshift; + case T_xoreq: + return OP_bit_xor; + case T_oreq: + return OP_bit_or; + case T_andeq: + return OP_bit_and; + default: + return OP_generic; + } +} + +static bool assignment_operator(token_kind_t kind) +{ + return kind == T_assign || compound_assignment_opcode(kind) != OP_generic; +} + +static bool assignment_operator_follows(void) +{ + token_t *token = cur_token->next; + + return token && assignment_operator(token->kind); +} + /* Consume a compound-assignment operator ("+=", "-=", ...) and report the * arithmetic it applies. * @@ -2241,40 +1685,30 @@ bool is_logical(opcode_t op) */ bool accept_compound_assign_op(opcode_t *op) { - if (lex_accept(T_pluseq)) - op[0] = OP_add; - else if (lex_accept(T_minuseq)) - op[0] = OP_sub; - else if (lex_accept(T_asteriskeq)) - op[0] = OP_mul; - else if (lex_accept(T_divideeq)) - op[0] = OP_div; - else if (lex_accept(T_modeq)) - op[0] = OP_mod; - else if (lex_accept(T_lshifteq)) - op[0] = OP_lshift; - else if (lex_accept(T_rshifteq)) - op[0] = OP_rshift; - else if (lex_accept(T_xoreq)) - op[0] = OP_bit_xor; - else if (lex_accept(T_oreq)) - op[0] = OP_bit_or; - else if (lex_accept(T_andeq)) - op[0] = OP_bit_and; - else + token_t *token = cur_token->next; + opcode_t opcode; + + if (!token || + (opcode = compound_assignment_opcode(token->kind)) == OP_generic || + !lex_accept(token->kind)) return false; + op[0] = opcode; return true; } +static bool accept_assignment_op(opcode_t *op) +{ + if (lex_accept(T_assign)) { + op[0] = OP_generic; + return true; + } + return accept_compound_assign_op(op); +} + bool lvalue_write_follows(opcode_t prefix_op) { - return prefix_op != OP_generic || lex_peek(T_assign, NULL) || - lex_peek(T_increment, NULL) || lex_peek(T_decrement, NULL) || - lex_peek(T_pluseq, NULL) || lex_peek(T_minuseq, NULL) || - lex_peek(T_asteriskeq, NULL) || lex_peek(T_divideeq, NULL) || - lex_peek(T_modeq, NULL) || lex_peek(T_lshifteq, NULL) || - lex_peek(T_rshifteq, NULL) || lex_peek(T_xoreq, NULL) || - lex_peek(T_oreq, NULL) || lex_peek(T_andeq, NULL); + return prefix_op != OP_generic || assignment_operator_follows() || + lex_peek(T_increment, NULL) || lex_peek(T_decrement, NULL); } int get_pointer_element_size(var_t *ptr_var) @@ -2338,25 +1772,22 @@ int get_pointer_element_size(var_t *ptr_var) * void pointer (void **) is different: its elements are pointer objects and * therefore have a known size. */ +bool is_direct_void_pointer_type(const type_t *type, int ptr_level) +{ + return type && type->base_type == TYPE_void && + (ptr_level == 1 || (ptr_level == 0 && type->ptr_level == 1)); +} + bool is_direct_void_pointer(const var_t *var) { - if (!var || !var->type || var->type->base_type != TYPE_void) + if (!var || var->array_size || var->has_unsized_array) return false; /* An array of void pointers decays to void ** before arithmetic. Its * elements are complete pointer objects, even though the declared base type * and direct declarator depth otherwise resemble void *. */ - if (var->array_size || var->has_unsized_array) - return false; - return var->ptr_level == 1 || - (var->ptr_level == 0 && var->type->ptr_level == 1); -} - -bool is_direct_void_pointer_type(const type_t *type, int ptr_level) -{ - return type && type->base_type == TYPE_void && - (ptr_level == 1 || (ptr_level == 0 && type->ptr_level == 1)); + return is_direct_void_pointer_type(var->type, var->ptr_level); } /* Describe in @decayed the pointer to its first row that the multidimensional @@ -2393,13 +1824,22 @@ static var_t *decay_function_pointer_array(block_t *parent, if (!var || !var->is_func || !var->func_signature || var->array_dim2 || !(var->array_size || var->has_unsized_array)) return var; - pointer = require_typed_ptr_var(parent, var->type, 1); - pointer->var_name = gen_name(); + pointer = name_var(require_typed_ptr_var(parent, var->type, 1)); pointer->pointee_func_signature = var->func_signature; add_insn(parent, *bb, OP_assign, pointer, var, NULL, 0, NULL); return pointer; } +static int pointer_arithmetic_stride(var_t *pointer) +{ + if (is_pointee_array_pointer(pointer) && !is_array_declarator(pointer)) + return pointee_array_row_stride(pointer); + if (is_array_declarator(pointer) && pointer->array_dim2 && + !has_effective_pointer(pointer) && !pointer->is_func) + return fixed_array_decay_stride(pointer); + return get_pointer_element_size(pointer); +} + /* Helper function to handle pointer arithmetic (add/sub with scaling) */ void handle_pointer_arithmetic(block_t *parent, basic_block_t **bb, @@ -2490,39 +1930,29 @@ void handle_pointer_arithmetic(block_t *parent, bool right_row = is_pointee_array_pointer(orig_rs2) && !is_array_declarator(orig_rs2); - if ((left_row || right_row) && - !pointee_array_shapes_compatible(orig_rs1, orig_rs2)) - error_at( - "Pointer subtraction requires compatible pointed-to types", - cur_token_loc()); - if (!compatible_decl_type(left_pointee, right_pointee) || + if (((left_row || right_row) && + !pointee_array_shapes_compatible(orig_rs1, orig_rs2)) || + !compatible_decl_type(left_pointee, right_pointee) || left_depth != right_depth) error_at( "Pointer subtraction requires compatible pointed-to types", cur_token_loc()); - element_size = PTR_SIZE; /* Default */ - - element_size = left_row ? pointee_array_row_stride(orig_rs1) - : get_pointer_element_size(orig_rs1); + element_size = pointer_arithmetic_stride(orig_rs1); /* Perform subtraction first */ - var_t *diff = require_var(parent); - diff->var_name = gen_name(); + var_t *diff = require_named_var(parent); add_insn(parent, *bb, OP_sub, diff, rs1, rs2, 0, NULL); /* Then divide by element size if needed */ if (element_size > 1) { - var_t *size_const = require_var(parent); - size_const->var_name = gen_name(); - size_const->init_val = element_size; - add_insn(parent, *bb, OP_load_constant, size_const, NULL, NULL, - 0, NULL); - - var_t *result = require_var(parent); - result->var_name = gen_name(); - add_insn(parent, *bb, OP_div, result, diff, size_const, 0, - NULL); + int shift = exact_log2(element_size); + var_t *size_const = load_constant( + parent, *bb, shift >= 0 ? shift : element_size, TY_int); + + var_t *result = require_named_var(parent); + add_insn(parent, *bb, shift >= 0 ? OP_rshift : OP_div, result, + diff, size_const, 0, NULL); opstack_push(result); } else { opstack_push(diff); @@ -2534,53 +1964,38 @@ void handle_pointer_arithmetic(block_t *parent, if (is_pointer_like_value(rs1)) { ptr_var = rs1; int_var = rs2; - element_size = - is_pointee_array_pointer(rs1) && !is_array_declarator(rs1) - ? pointee_array_row_stride(rs1) - : (is_array_declarator(rs1) && rs1->array_dim2 && - !has_effective_pointer(rs1) && !rs1->is_func - ? fixed_array_decay_stride(rs1) - : get_pointer_element_size(rs1)); + element_size = pointer_arithmetic_stride(rs1); } else if (is_pointer_like_value(rs2)) { /* Only for addition (p + n == n + p) */ if (op == OP_add) { ptr_var = rs2; int_var = rs1; - element_size = - is_pointee_array_pointer(rs2) && !is_array_declarator(rs2) - ? pointee_array_row_stride(rs2) - : (is_array_declarator(rs2) && rs2->array_dim2 && - !has_effective_pointer(rs2) && !rs2->is_func - ? fixed_array_decay_stride(rs2) - : get_pointer_element_size(rs2)); + element_size = pointer_arithmetic_stride(rs2); /* Swap operands so pointer is rs1 */ rs1 = ptr_var; rs2 = int_var; } } + if (ptr_var && int_var->is_const && !int_var->is_global && + !int_var->init_val && !int_var->init_val_hi && !ptr_var->array_size && + !ptr_var->has_unsized_array) { + opstack_push(ptr_var); + return; + } + /* If we need to scale the integer operand */ if (ptr_var && element_size > 1) { - /* Create multiplication by element size */ - var_t *size_const = require_var(parent); - size_const->var_name = gen_name(); - size_const->init_val = element_size; - add_insn(parent, *bb, OP_load_constant, size_const, NULL, NULL, 0, - NULL); - - var_t *scaled = require_var(parent); - scaled->var_name = gen_name(); - add_insn(parent, *bb, OP_mul, scaled, int_var, size_const, 0, NULL); - - /* Use scaled value as rs2 */ - rs2 = scaled; + rs2 = scale_pointer_index(parent, *bb, int_var, element_size, true); } /* Perform the operation */ - var_t *vd = require_var(parent); + var_t *vd = name_var(require_var(parent)); /* Preserve pointer type metadata on results of pointer arithmetic */ if (ptr_var) { vd->type = ptr_var->type; + vd->is_volatile_access = + ptr_var->is_volatile_access || var_volatile_pointee(ptr_var); /* An array operand decays to a pointer before arithmetic. Retain that * extra level on the result so `slots + 0` for an array of pointer @@ -2610,7 +2025,6 @@ void handle_pointer_arithmetic(block_t *parent, fixed_array_shape_to_pointee_var(vd, &shape); } } - vd->var_name = gen_name(); opstack_push(vd); add_insn(parent, *bb, op, vd, rs1, rs2, 0, NULL); } @@ -2753,6 +2167,36 @@ void normalize_integer_binary_operands(block_t *parent, right[0] = resize_to(parent, bb, right[0], common, 0); } +static var_t *emit_binary_result(block_t *parent, + basic_block_t **bb, + opcode_t op, + var_t *left, + var_t *right, + type_t *result_type) +{ + var_t *result = require_named_var(parent); + result->type = result_type; + add_insn(parent, *bb, op, result, left, right, 0, NULL); + return result; +} + +static var_t *combine_assignment_value(block_t *parent, + basic_block_t **bb, + opcode_t op, + var_t *current, + var_t *value) +{ + if (is_pointer_operation(op, current, value)) { + handle_pointer_arithmetic(parent, bb, op, current, value); + return opstack_pop(); + } + current = integer_promote_operand(parent, bb, current); + value = integer_promote_operand(parent, bb, value); + normalize_integer_binary_operands(parent, bb, op, ¤t, &value); + return emit_binary_result(parent, bb, op, current, value, + integer_binary_result_type(op, current, value)); +} + bool emit_wide_global_word_arithmetic(block_t *parent, basic_block_t *bb, var_t *result, @@ -2764,8 +2208,9 @@ bool emit_wide_global_word_arithmetic(block_t *parent, * alone. Fold those in two-word form before the usual arithmetic conversions * emit IR that hides the constants, so an unsigned expression such as sizeof(x) * - sizeof(x) stays an integer constant expression and, when zero, a null - * pointer constant. Division by zero and a shift outside the operand width are - * left to the target, as the int folder leaves them. + * pointer constant. Division by zero, signed minimum/-1 overflow, and shifts + * outside the operand width are left to the target, as the int folder leaves + * them. */ static bool fold_wide_constant_binary(block_t *parent, basic_block_t **bb, @@ -2784,43 +2229,45 @@ static bool fold_wide_constant_binary(block_t *parent, rs1->type->size <= TY_int->size && rs2->type->size <= TY_int->size) return false; - switch (op) { - case OP_div: - case OP_mod: - if (!rs2->init_val && - (rs2->type->size <= TY_int->size || !rs2->init_val_hi)) - return false; - break; - case OP_lshift: - case OP_rshift: - if ((rs2->type->size > TY_int->size && rs2->init_val_hi) || - rs2->init_val < 0 || rs2->init_val >= rs1->type->size * 8) - return false; - break; - case OP_add: - case OP_sub: - case OP_mul: - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: - case OP_eq: - case OP_neq: - case OP_lt: - case OP_leq: - case OP_gt: - case OP_geq: - break; - default: + if (!op_is_binary_alu(op)) + return false; + if ((op == OP_div || op == OP_mod) && !rs2->init_val && + (rs2->type->size <= TY_int->size || !rs2->init_val_hi)) + return false; + if ((op == OP_lshift || op == OP_rshift) && + ((rs2->type->size > TY_int->size && rs2->init_val_hi) || + rs2->init_val < 0 || rs2->init_val >= rs1->type->size * 8)) + return false; + if ((op == OP_div || op == OP_mod) && !unsigned_int_operand(rs1) && + !unsigned_int_operand(rs2) && rs1->type->size > TY_int->size && + rs1->init_val == 0 && rs1->init_val_hi == INT_MIN && + rs2->init_val == -1 && + (rs2->type->size <= TY_int->size || rs2->init_val_hi == -1)) return false; - } - vd = require_typed_var(parent, integer_binary_result_type(op, rs1, rs2)); - vd->var_name = gen_name(); + vd = name_var( + require_typed_var(parent, integer_binary_result_type(op, rs1, rs2))); emit_wide_global_word_arithmetic(parent, *bb, vd, op, rs1, rs2); opstack_push(vd); return true; } +static void pop_binary_operands(block_t *parent, + basic_block_t **bb, + opcode_t op, + var_t **left, + var_t **right) +{ + *right = opstack_pop(); + *left = opstack_pop(); + reject_record_operand(*left); + reject_record_operand(*right); + if (op == OP_eq || op == OP_neq) { + *left = materialize_function_designator(parent, bb, *left); + *right = materialize_function_designator(parent, bb, *right); + } +} + void read_expr_body(block_t *parent, basic_block_t **bb) { var_t *vd, *rs1, *rs2; @@ -2837,8 +2284,7 @@ void read_expr_body(block_t *parent, basic_block_t **bb) */ bool has_prev_log_op = false; opcode_t prev_log_op = 0, pprev_log_op = 0; - basic_block_t *log_and_shared_bb = bb_create(parent), - *log_or_shared_bb = bb_create(parent); + basic_block_t *log_and_shared_bb = NULL, *log_or_shared_bb = NULL; read_expr_operand(parent, bb); @@ -2846,6 +2292,11 @@ void read_expr_body(block_t *parent, basic_block_t **bb) if (op == OP_generic || op == OP_ternary) return; if (is_logical(op)) { + /* Most expressions have no logical operator, so the shared blocks are + * created on the first one. + */ + log_and_shared_bb = bb_create(parent); + log_or_shared_bb = bb_create(parent); bb_connect(*bb, op == OP_log_and ? log_and_shared_bb : log_or_shared_bb, op == OP_log_and ? ELSE : THEN); read_logical(op, parent, bb); @@ -2865,40 +2316,15 @@ void read_expr_body(block_t *parent, basic_block_t **bb) do { opcode_t top_op = oper_stack[oper_stack_idx - 1]; if (get_operator_prio(top_op) >= get_operator_prio(op)) { - rs2 = opstack_pop(); - rs1 = opstack_pop(); - reject_record_operand(rs1); - reject_record_operand(rs2); + pop_binary_operands(parent, bb, top_op, &rs1, &rs2); /* As in the final reduction below, equality compares a * function designator as its pointer. Reduced here, before * a following operator such as ||, a raw symbol compared * as garbage. */ - if (top_op == OP_eq || top_op == OP_neq) { - rs1 = materialize_function_designator(parent, bb, rs1); - rs2 = materialize_function_designator(parent, bb, rs2); - } - - /* Handle pointer arithmetic for addition and subtraction */ - if (is_pointer_operation(top_op, rs1, rs2)) { - /* handle_pointer_arithmetic handles both pointer - * differences and regular pointer arithmetic internally - */ - handle_pointer_arithmetic(parent, bb, top_op, rs1, rs2); - oper_stack_idx--; - continue; - } - - rs1 = integer_promote_operand(parent, bb, rs1); - rs2 = integer_promote_operand(parent, bb, rs2); - normalize_integer_binary_operands(parent, bb, top_op, &rs1, - &rs2); - vd = require_var(parent); - vd->var_name = gen_name(); - vd->type = integer_binary_result_type(top_op, rs1, rs2); + vd = combine_assignment_value(parent, bb, top_op, rs1, rs2); opstack_push(vd); - add_insn(parent, *bb, top_op, vd, rs1, rs2, 0, NULL); oper_stack_idx--; } else @@ -2906,6 +2332,10 @@ void read_expr_body(block_t *parent, basic_block_t **bb) } while (oper_stack_idx > 0 && same == 0); } if (is_logical(op)) { + if (!log_and_shared_bb) { + log_and_shared_bb = bb_create(parent); + log_or_shared_bb = bb_create(parent); + } if (prev_log_op == 0 || prev_log_op == op) { bb_connect( *bb, @@ -3008,23 +2438,15 @@ void read_expr_body(block_t *parent, basic_block_t **bb) while (oper_stack_idx > 0) { opcode_t top_op = oper_stack[--oper_stack_idx]; - rs2 = opstack_pop(); - rs1 = opstack_pop(); - reject_record_operand(rs1); - reject_record_operand(rs2); + pop_binary_operands(parent, bb, top_op, &rs1, &rs2); /* Equality compares function pointers after the C99 function-to- * pointer conversion. A raw symbol has no SSA value and otherwise looks * like zero to the backend, making `function == 0` spuriously true. * Other arithmetic operations retain their function-pointer constraint - * diagnostics below. + * diagnostics below. An array of function pointers decays to an object + * pointer. */ - if (top_op == OP_eq || top_op == OP_neq) { - rs1 = materialize_function_designator(parent, bb, rs1); - rs2 = materialize_function_designator(parent, bb, rs2); - } - - /* An array of function pointers decays to an object pointer. */ if ((top_op == OP_lt || top_op == OP_leq || top_op == OP_gt || top_op == OP_geq) && ((rs1 && !is_array_declarator(rs1) && @@ -3098,88 +2520,13 @@ void read_expr_body(block_t *parent, basic_block_t **bb) !unsigned_int_operand(rs1) && !unsigned_int_operand(rs2) && rs1->type->size <= TY_int->size && rs2->type->size <= TY_int->size) { - /* Both operands are compile-time constants */ - int result = 0; - bool folded = true; - - switch (top_op) { - case OP_add: - result = rs1->init_val + rs2->init_val; - break; - case OP_sub: - result = rs1->init_val - rs2->init_val; - break; - case OP_mul: - result = rs1->init_val * rs2->init_val; - break; - case OP_div: - if (rs2->init_val != 0) - result = rs1->init_val / rs2->init_val; - else - folded = false; /* Division by zero */ - break; - case OP_mod: - if (rs2->init_val != 0) - result = rs1->init_val % rs2->init_val; - else - folded = false; /* Modulo by zero */ - break; - case OP_bit_and: - result = rs1->init_val & rs2->init_val; - break; - case OP_bit_or: - result = rs1->init_val | rs2->init_val; - break; - case OP_bit_xor: - result = rs1->init_val ^ rs2->init_val; - break; - - /* A shift count outside the promoted int width has no defined - * result, and the compiler must not compute one in the host either: - * leave such a shift to the target. Otherwise shift the unsigned - * bit pattern, so a bit reaching the sign is not host undefined - * behavior. - */ - case OP_lshift: - if (rs2->init_val < 0 || rs2->init_val >= 32) - folded = false; - else - result = - (int) ((unsigned int) rs1->init_val << rs2->init_val); - break; - case OP_rshift: - if (rs2->init_val < 0 || rs2->init_val >= 32) - folded = false; - else - result = rs1->init_val >> rs2->init_val; - break; - case OP_eq: - result = rs1->init_val == rs2->init_val; - break; - case OP_neq: - result = rs1->init_val != rs2->init_val; - break; - case OP_lt: - result = rs1->init_val < rs2->init_val; - break; - case OP_leq: - result = rs1->init_val <= rs2->init_val; - break; - case OP_gt: - result = rs1->init_val > rs2->init_val; - break; - case OP_geq: - result = rs1->init_val >= rs2->init_val; - break; - default: - folded = false; - break; - } + int result; + bool folded = vir_frontend_fold_i32(top_op, rs1->init_val, + rs2->init_val, &result); if (folded) { /* Create constant result */ - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); vd->type = result_type; vd->is_const = true; vd->init_val = result; @@ -3195,19 +2542,14 @@ void read_expr_body(block_t *parent, basic_block_t **bb) NULL); } else { /* Normal operation - folding failed or not supported */ - vd = require_var(parent); - vd->var_name = gen_name(); - vd->type = result_type; + vd = emit_binary_result(parent, bb, top_op, rs1, rs2, + result_type); opstack_push(vd); - add_insn(parent, *bb, top_op, vd, rs1, rs2, 0, NULL); } } else { /* Normal operation */ - vd = require_var(parent); - vd->var_name = gen_name(); - vd->type = result_type; + vd = emit_binary_result(parent, bb, top_op, rs1, rs2, result_type); opstack_push(vd); - add_insn(parent, *bb, top_op, vd, rs1, rs2, 0, NULL); } } while (has_prev_log_op) { @@ -3308,6 +2650,88 @@ static int lvalue_step_size(const lvalue_t *lvalue, var_t *var) return 1; } +static void set_lvalue_pointer_pointee(lvalue_t *lvalue) +{ + type_t *pointee = pointee_type_from_pointer_typedef(lvalue->type); + + lvalue->size = pointer_typedef_pointee_size(lvalue->type, pointee); + lvalue->type = pointee; +} + +static var_t *read_addressed_value(block_t *parent, + basic_block_t **bb, + var_t *address, + const lvalue_t *lvalue, + bool assignment_result, + bool removable) +{ + var_t *value; + + if (is_bitfield(lvalue->decl)) + return assignment_result + ? reload_assigned_bitfield(parent, bb, address, lvalue->decl) + : read_bitfield_value(parent, bb, address, lvalue->decl); + value = name_var( + require_typed_ptr_var(parent, lvalue->type, lvalue->value_ptr_level)); + value->is_assignment_reload = removable && assignment_result; + add_insn(parent, *bb, OP_read, value, address, NULL, lvalue->size, NULL); + return value; +} + +static var_t *combine_addressed_assignment(block_t *parent, + basic_block_t **bb, + opcode_t op, + var_t *address, + const lvalue_t *target, + var_t *value) +{ + return op == OP_generic ? value + : combine_assignment_value( + parent, bb, op, + read_addressed_value(parent, bb, address, + target, false, false), + value); +} + +static var_t *store_addressed_assignment(block_t *parent, + basic_block_t **bb, + var_t *address, + const lvalue_t *target, + var_t *value, + bool use_resize_to) +{ + if (is_bitfield(target->decl)) { + write_bitfield_value(parent, bb, address, value, target->decl); + return value; + } + if (use_resize_to) { + if (!value->is_func) + value = resize_to(parent, bb, value, target->type, + target->value_ptr_level); + } else + value = convert_stored_value(parent, bb, value, target->type, + target->value_ptr_level); + add_insn(parent, *bb, OP_write, NULL, address, value, target->size, NULL); + return value; +} + +static var_t *read_lvalue_update_value(block_t *parent, + basic_block_t **bb, + const lvalue_t *lvalue) +{ + var_t *address; + var_t *value; + + if (!lvalue->is_reference) + return operand_stack[operand_stack_idx - 1]; + address = opstack_pop(); + lvalue_t target = *lvalue; + target.type = pointee_type_from_pointer_typedef(lvalue->type); + value = read_addressed_value(parent, bb, address, &target, false, false); + opstack_push(value); + return address; +} + /* What follows a completed lvalue: an update or read of the object it names. * read_lvalue() finishes with this. */ @@ -3331,12 +2755,16 @@ static void lower_lvalue_tail(lvalue_t *lvalue, */ if (lvalue->is_reference) { rs1 = operand_stack[operand_stack_idx - 1]; - t = require_var(parent); - t->var_name = gen_name(); - t->type = pointer_row_element_type - ? pointer_row_element_type - : pointee_type_from_pointer_typedef(lvalue->type); + t = require_named_var(parent); + t->type = pointee_type_from_pointer_typedef( + pointer_row_element_type ? pointer_row_element_type : lvalue->type); t->ptr_level = lvalue->value_ptr_level; + t->is_volatile = var->is_volatile; + int depth = t->ptr_level + t->type->ptr_level; + t->pointer_volatile_mask = + depth > 0 && depth <= 32 + ? var->pointer_volatile_mask & ~(1U << (depth - 1)) + : 0; if (pointer_row_element_type) t->is_const_qualified = lvalue->type->is_const_qualified; @@ -3354,16 +2782,26 @@ static void lower_lvalue_tail(lvalue_t *lvalue, if (lvalue->decl && lvalue->decl->pointee_array_size && !lvalue->pointee_func_signature && (is_array_declarator(lvalue->decl) - ? lvalue->decl->ptr_level == 1 && lvalue->subscript_depth == 1 + ? effective_pointer_depth(lvalue->decl) > + lvalue->decl->pointee_array_element_ptr_level && + lvalue->subscript_depth == 1 : !lvalue->subscript_depth || - lvalue->subscript_depth == lvalue->decl->ptr_level - 1)) + lvalue->subscript_depth == + lvalue->decl->ptr_level - 1)) { copy_pointee_array_shape(t, lvalue->decl); + t->pointer_const_mask = + dereferenced_pointer_const_mask(lvalue->decl); + if (depth < 32) + t->pointer_const_mask &= (1U << depth) - 1; + } /* Retain a callback prototype even through a selected slot. A direct * loaded callback is callable, while unary `*` restores this marker * after consuming an extra object-pointer level. */ t->func_signature = lvalue->type->func_signature; + if (t->func_signature && !t->ptr_level) + t->ptr_level = 1; /* `fpp[i]` of an `int (**fpp)(int)` loads a callable function pointer, * and so do `fps[i]` of an `int (*fps[N])(int)` and a member `s.fp`. @@ -3419,25 +2857,19 @@ static void lower_lvalue_tail(lvalue_t *lvalue, if ((prefix_op == OP_add || prefix_op == OP_sub) && is_direct_void_pointer_type(lvalue->type, lvalue->ptr_level)) error_at("Pointer arithmetic on void* is invalid", cur_token_loc()); - vd = require_var(parent); - vd->var_name = gen_name(); - - vd->init_val = lvalue_step_size(lvalue, var); - if (lvalue_is_wide_integer(lvalue)) - vd->type = lvalue->type; + vd = load_constant( + parent, *bb, lvalue_step_size(lvalue, var), + lvalue_is_wide_integer(lvalue) ? lvalue->type : TY_int); opstack_push(vd); - add_insn(parent, *bb, OP_load_constant, vd, NULL, NULL, 0, NULL); rs2 = opstack_pop(); if (lvalue->is_reference) rs1 = opstack_pop(); else rs1 = operand_stack[operand_stack_idx - 1]; - vd = require_var(parent); - vd->var_name = gen_name(); - if (lvalue_is_wide_integer(lvalue)) - vd->type = lvalue->type; - add_insn(parent, *bb, prefix_op, vd, rs1, rs2, 0, NULL); + vd = emit_binary_result( + parent, bb, prefix_op, rs1, rs2, + lvalue_is_wide_integer(lvalue) ? lvalue->type : TY_int); if (lvalue->is_reference) { rs1 = vd; @@ -3455,9 +2887,6 @@ static void lower_lvalue_tail(lvalue_t *lvalue, rs1 = convert_stored_value(parent, bb, rs1, t->type, t->ptr_level); - /* The column of arguments of the new insn of 'OP_write' is - * different from 'ph1_ir' - */ add_insn(parent, *bb, OP_write, NULL, vd, rs1, lvalue->size, NULL); } @@ -3483,17 +2912,11 @@ static void lower_lvalue_tail(lvalue_t *lvalue, /* Consume the '--' as well: testing for '++' alone left it in the * stream, so `bits.field--` failed to parse. */ - opcode_t postfix_op = OP_sub; - if (lex_accept(T_increment)) - postfix_op = OP_add; - else - lex_expect(T_decrement); + opcode_t postfix_op = read_increment_operator(); var_t *old = opstack_pop(); var_t *address = opstack_pop(); var_t *one = bitfield_constant(parent, bb, 1); - var_t *updated = require_var(parent); - - updated->var_name = gen_name(); + var_t *updated = require_named_var(parent); updated->type = integer_binary_result_type(postfix_op, old, one); add_insn(parent, *bb, postfix_op, updated, old, one, 0, NULL); write_bitfield_value(parent, bb, address, updated, lvalue->decl); @@ -3508,13 +2931,9 @@ static void lower_lvalue_tail(lvalue_t *lvalue, * operator, and before a call (C99 6.5.2.2p10). Deferring it to the end * of the statement also ran updates in a skipped operand. */ - vd = require_var(parent); - vd->var_name = gen_name(); - - vd->init_val = lvalue_step_size(lvalue, var); - if (lvalue_is_wide_integer(lvalue)) - vd->type = lvalue->type; - add_insn(parent, *bb, OP_load_constant, vd, NULL, NULL, 0, NULL); + vd = load_constant( + parent, *bb, lvalue_step_size(lvalue, var), + lvalue_is_wide_integer(lvalue) ? lvalue->type : TY_int); rs2 = vd; /* Consume whichever operator is actually there. Testing only for '++' @@ -3522,11 +2941,7 @@ static void lower_lvalue_tail(lvalue_t *lvalue, * decrement parsed only where the leftover token happened to be * harmless -- "i--;" as a statement worked, "a = i--" did not. */ - opcode_t postfix_op = OP_sub; - if (lex_accept(T_increment)) - postfix_op = OP_add; - else - lex_expect(T_decrement); + opcode_t postfix_op = read_increment_operator(); /* The expression has the value the object held before the update. A * reference was already loaded into a temporary; a variable is itself @@ -3537,8 +2952,7 @@ static void lower_lvalue_tail(lvalue_t *lvalue, rs1 = opstack_pop(); } else { rs1 = operand_stack[operand_stack_idx - 1]; - t = require_var(parent); - t->var_name = gen_name(); + t = require_named_var(parent); t->type = rs1->type; t->ptr_level = rs1->ptr_level; t->func_signature = rs1->func_signature; @@ -3548,8 +2962,7 @@ static void lower_lvalue_tail(lvalue_t *lvalue, copy_pointee_array_shape(t, rs1); add_insn(parent, *bb, OP_assign, t, rs1, NULL, 0, NULL); } - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); if (!lvalue->is_reference && is_pointee_array_pointer(var)) { vd->type = lvalue->type; vd->ptr_level = var->ptr_level; @@ -3617,7 +3030,7 @@ static bool is_function_parameter(const var_t *var, block_t *parent) for (int i = 0; i < parent->func->num_params; i++) { var_t *param = &parent->func->param_defs[i]; - if (var == param || var->base == param) + if (var == param) return true; } return false; @@ -3659,6 +3072,8 @@ void read_lvalue(lvalue_t *lvalue, * qualification; the object a pointer member points to does not. */ bool record_is_const = false; + bool record_is_volatile = false; + bool object_is_volatile = var_volatile_storage(var); /* Callers pass a find_var() result, which is NULL for a name that was never * declared. @@ -3702,7 +3117,7 @@ void read_lvalue(lvalue_t *lvalue, lvalue->pointer_const_mask = var->pointer_const_mask; opstack_push(var); - if (var->is_volatile) + if (var_is_volatile_object(var)) unread_volatile_object = var; if (lex_peek(T_open_square, NULL) || lex_peek(T_arrow, NULL) || @@ -3713,6 +3128,9 @@ void read_lvalue(lvalue_t *lvalue, lex_peek(T_dot, NULL)) { array_row = NULL; if (lex_accept(T_open_square)) { + object_is_volatile = + var_volatile_pointee(var) || + (record_is_volatile && is_array_declarator(var)); int indexed_ptr_level; fixed_array_shape_t row_shape; bool indexes_fixed_array_pointer_slot; @@ -3730,8 +3148,9 @@ void read_lvalue(lvalue_t *lvalue, (lvalue->value_ptr_level || pending_fixed_array_pointer_slot) && is_member) { rs1 = opstack_pop(); - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); + vd->type = pointee_type_from_pointer_typedef(lvalue->type); + vd->ptr_level = lvalue->value_ptr_level; if (pending_scalar_row_pointer_slot) { vd->type = var->type; vd->ptr_level = 1; @@ -3765,12 +3184,7 @@ void read_lvalue(lvalue_t *lvalue, lvalue->size = PTR_SIZE; } else if (lvalue->type->ptr_level && !loaded_scalar_row && !lvalue->type->pointee_array_size) { - type_t *pointee = - pointee_type_from_pointer_typedef(lvalue->type); - - lvalue->size = - pointer_typedef_pointee_size(lvalue->type, pointee); - lvalue->type = pointee; + set_lvalue_pointer_pointee(lvalue); } else { lvalue->size = lvalue->type->size; } @@ -3849,7 +3263,7 @@ void read_lvalue(lvalue_t *lvalue, pointer_row_element_type = var->type->pointee_array_element_type ? var->type->pointee_array_element_type - : var->type; + : array_typedef_element_type(var->type); } else if (!subscript_depth && is_array_declarator(var) && var->type->array_element_ptr_level > 0) { pointer_row_element_type = var->type->array_element_type @@ -3870,17 +3284,12 @@ void read_lvalue(lvalue_t *lvalue, * pointer points to */ if (lvalue->type->ptr_level > 0) { - type_t *pointee = - pointee_type_from_pointer_typedef(lvalue->type); - /* void pointers retain the existing byte-stride extension; * every other typedef pointer advances by its actual * pointee, including a tagged record reached through a * typedef alias. */ - lvalue->size = - pointer_typedef_pointee_size(lvalue->type, pointee); - lvalue->type = pointee; + set_lvalue_pointer_pointee(lvalue); } else { lvalue->size = lvalue->type->size; } @@ -3904,17 +3313,15 @@ void read_lvalue(lvalue_t *lvalue, if (subscript_depth == 0 && var->array_dim2 > 0) { multiplier = fixed_array_subscript_stride(var, subscript_depth, lvalue->size); - } else if (subscript_depth == 0 && var->pointee_array_size > 0 && - indexes_pointee_row) { - multiplier = fixed_pointee_array_subscript_stride( - var, subscript_depth, lvalue->size); - } else if (indexes_loaded_scalar_pointee_row) { - multiplier = fixed_pointee_array_subscript_stride( - loaded_scalar_row, 0, lvalue->size); } else if (loaded_scalar_row) { + int row_depth = + indexes_loaded_scalar_pointee_row ? 0 : subscript_depth - 1; + multiplier = fixed_pointee_array_subscript_stride( - loaded_scalar_row, subscript_depth - 1, lvalue->size); - } else if (subscript_depth >= 1 && var->pointee_array_size > 0) { + loaded_scalar_row, row_depth, lvalue->size); + } else if (var->pointee_array_size > 0 && + ((subscript_depth == 0 && indexes_pointee_row) || + subscript_depth >= 1)) { multiplier = fixed_pointee_array_subscript_stride( var, subscript_depth, lvalue->size); } else if ((subscript_depth == 1 && var->array_dim3 > 0) || @@ -3924,24 +3331,14 @@ void read_lvalue(lvalue_t *lvalue, } if (multiplier != 1) { - vd = require_var(parent); - vd->init_val = multiplier; - vd->var_name = gen_name(); - opstack_push(vd); - add_insn(parent, *bb, OP_load_constant, vd, NULL, NULL, 0, - NULL); - - rs2 = opstack_pop(); rs1 = opstack_pop(); - vd = require_var(parent); - vd->var_name = gen_name(); - opstack_push(vd); - add_insn(parent, *bb, OP_mul, vd, rs1, rs2, 0, NULL); + opstack_push( + scale_pointer_index(parent, *bb, rs1, multiplier, false)); } rs2 = opstack_pop(); rs1 = opstack_pop(); - vd = require_var(parent); + vd = name_var(require_var(parent)); /* A subscript expression computes the address of its selected * element. Preserve that pointer provenance: unary '&' leaves an @@ -3956,7 +3353,7 @@ void read_lvalue(lvalue_t *lvalue, * `int **`, not `int *`. */ vd->ptr_level = lvalue->ptr_level + 1; - vd->var_name = gen_name(); + vd->is_volatile_access = object_is_volatile; opstack_push(vd); add_insn(parent, *bb, OP_add, vd, rs1, rs2, 0, NULL); @@ -3975,8 +3372,7 @@ void read_lvalue(lvalue_t *lvalue, for (int i = 0; i < subscript_depth + 2; i++) fixed_array_shape_drop_outer(&row_shape); if (row_shape.rank) { - fixed_array_shape_to_pointee_var(vd, &row_shape); - vd->pointee_array_element_ptr_level = var->ptr_level; + set_pointee_array_shape(vd, &row_shape, var->ptr_level); } vd->is_const_qualified = var->is_const_qualified; lvalue->value_ptr_level = indexed_ptr_level; @@ -3999,9 +3395,9 @@ void read_lvalue(lvalue_t *lvalue, vd->ptr_level = row_source->pointee_array_element_ptr_level + 1; if (row_shape.rank) { - fixed_array_shape_to_pointee_var(vd, &row_shape); - vd->pointee_array_element_ptr_level = - row_source->pointee_array_element_ptr_level; + set_pointee_array_shape( + vd, &row_shape, + row_source->pointee_array_element_ptr_level); } vd->is_const_qualified = var->is_const_qualified; lvalue->value_ptr_level = indexed_ptr_level; @@ -4069,14 +3465,16 @@ void read_lvalue(lvalue_t *lvalue, * pointer's declaration however the pointer itself is. */ record_is_const = var->is_const_qualified; + record_is_volatile = var_volatile_pointee(var); /* resolve where the pointer points at from the calculated * address in a structure. */ if (is_member) { rs1 = opstack_pop(); - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); + vd->type = pointee_type_from_pointer_typedef(lvalue->type); + vd->ptr_level = lvalue->value_ptr_level; opstack_push(vd); add_insn(parent, *bb, OP_read, vd, rs1, NULL, PTR_SIZE, NULL); @@ -4084,11 +3482,11 @@ void read_lvalue(lvalue_t *lvalue, } else { lex_expect(T_dot); record_is_const = lvalue->is_const_qualified; + record_is_volatile = object_is_volatile; if (!is_address_got) { rs1 = opstack_pop(); - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); opstack_push(vd); add_insn(parent, *bb, OP_address_of, vd, rs1, NULL, 0, NULL); @@ -4105,6 +3503,8 @@ void read_lvalue(lvalue_t *lvalue, error_at("Unknown struct or union member", next_token_loc()); lvalue->type = var->type; lvalue->decl = var; + object_is_volatile = + record_is_volatile || var_volatile_storage(var); /* The member, not the pointer row element it was selected from, now * types the value that is read. @@ -4133,16 +3533,15 @@ void read_lvalue(lvalue_t *lvalue, lvalue->is_reference = !is_array_declarator(var); /* move pointer to offset of structure */ - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); vd->init_val = var->offset; opstack_push(vd); add_insn(parent, *bb, OP_load_constant, vd, NULL, NULL, 0, NULL); rs2 = opstack_pop(); rs1 = opstack_pop(); - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); + vd->is_volatile_access = object_is_volatile; if (is_array_declarator(var)) { /* A one-dimensional array member designates its first element's @@ -4223,7 +3622,7 @@ void finalize_logical(opcode_t op, basic_block_t **bb, basic_block_t *shared_bb) { - basic_block_t *then, *then_next, *else_if, *else_bb; + basic_block_t *then, *then_next = NULL, *else_if, *else_bb; basic_block_t *end = bb_create(parent); var_t *vd, *log_op_res; @@ -4316,14 +3715,10 @@ void finalize_logical(opcode_t op, * Otherwise, create a false branch and assign a false value for logical-or * operation. */ - vd = require_var(parent); - vd->var_name = gen_name(); - vd->init_val = op == OP_log_and; - add_insn(parent, op == OP_log_and ? then_next : else_bb, OP_load_constant, - vd, NULL, NULL, 0, NULL); - - log_op_res = require_var(parent); - log_op_res->var_name = gen_name(); + vd = load_constant(parent, op == OP_log_and ? then_next : else_bb, + op == OP_log_and, TY_int); + + log_op_res = require_named_var(parent); add_insn(parent, op == OP_log_and ? then_next : else_bb, OP_assign, log_op_res, vd, NULL, 0, NULL); @@ -4337,11 +3732,8 @@ void finalize_logical(opcode_t op, * If handing a logical-and operation, assign a false value. else, assign a * true value for a logical-or operation. */ - vd = require_var(parent); - vd->var_name = gen_name(); - vd->init_val = op != OP_log_and; - add_insn(parent, op == OP_log_and ? else_bb : then, OP_load_constant, vd, - NULL, NULL, 0, NULL); + vd = load_constant(parent, op == OP_log_and ? else_bb : then, + op != OP_log_and, TY_int); add_insn(parent, op == OP_log_and ? else_bb : then, OP_assign, log_op_res, vd, NULL, 0, NULL); @@ -4358,18 +3750,7 @@ void finalize_logical(opcode_t op, */ void read_control_expression(block_t *parent, basic_block_t **bb) { - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - - while (lex_accept(T_comma)) { - discard_operand(parent, *bb); - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - } + read_comma_expression(parent, bb, false); /* A function designator in a controlling expression decays to its code * pointer before OP_branch tests it. Raw symbols intentionally have no @@ -4394,126 +3775,6 @@ basic_block_t *read_full_expression_statement(block_t *parent, return bb; } -/* This is deliberately narrower than expression assignment recognition: a - * grouped pointer-to-row store has no general lvalue representation yet. - */ -static bool grouped_scalar_pointee_row_store_starts(block_t *parent) -{ - token_t *token = cur_token ? cur_token->next : NULL; - int depth = 0; - - if (!token || token->kind != T_open_bracket || !(token = token->next) || - token->kind != T_asterisk || !(token = token->next) || - token->kind != T_identifier || - !names_scalar_row_pointer(token, parent) || !(token = token->next) || - token->kind != T_close_bracket || !(token = token->next) || - token->kind != T_open_square) - return false; - for (; token; token = token->next) { - if (token->kind == T_open_square || token->kind == T_open_bracket || - token->kind == T_open_curly) - depth++; - else if (token->kind == T_close_square || - token->kind == T_close_bracket || - token->kind == T_close_curly) { - if (!--depth) - break; - } - } - if (!token || !(token = token->next) || - (token->kind != T_assign && token->kind != T_pluseq && - token->kind != T_minuseq)) - return false; - for (depth = 0, token = token->next; token; token = token->next) { - if (token->kind == T_open_square || token->kind == T_open_bracket || - token->kind == T_open_curly) - depth++; - else if (token->kind == T_close_square || - token->kind == T_close_bracket || token->kind == T_close_curly) - depth--; - else if (!depth && token->kind == T_comma) - return false; - else if (!depth && token->kind == T_semicolon) - return true; - } - return false; -} - -static basic_block_t *handle_grouped_scalar_pointee_row_store(block_t *parent, - basic_block_t *bb) -{ - char name[MAX_VAR_LEN]; - var_t *source, *row, *index, *address, *value; - - lex_expect(T_open_bracket); - lex_expect(T_asterisk); - lex_ident(T_identifier, name); - source = find_var(name, parent); - if (!source) - error_at("Undeclared identifier", cur_token_loc()); - if (!lower_pointee_array_dereference(source, parent, &bb)) - error_at("Grouped pointer-to-array store requires a scalar fixed row", - cur_token_loc()); - row = opstack_pop(); - if (row->is_const_qualified) - error_at("assignment of read-only location", cur_token_loc()); - lex_expect(T_close_bracket); - lex_expect(T_open_square); - if (!read_assignment_expression(parent, &bb)) { - read_expr(parent, &bb); - read_ternary_operation(parent, &bb); - } - index = opstack_pop(); - lex_expect(T_close_square); - if (row->type->size != 1) { - var_t *scale = require_var(parent); - var_t *scaled = require_var(parent); - scale->var_name = gen_name(); - scale->init_val = row->type->size; - add_insn(parent, bb, OP_load_constant, scale, NULL, NULL, 0, NULL); - scaled->var_name = gen_name(); - add_insn(parent, bb, OP_mul, scaled, index, scale, 0, NULL); - index = scaled; - } - address = require_typed_ptr_var(parent, row->type, 1); - address->var_name = gen_name(); - add_insn(parent, bb, OP_add, address, row, index, 0, NULL); - opcode_t compound_op = OP_generic; - if (lex_accept(T_pluseq)) - compound_op = OP_add; - else if (lex_accept(T_minuseq)) - compound_op = OP_sub; - else - lex_expect(T_assign); - if (!read_assignment_expression(parent, &bb)) { - read_expr(parent, &bb); - read_ternary_operation(parent, &bb); - } - value = opstack_pop(); - if (compound_op != OP_generic) { - if (value->is_func || is_pointer_like_value(value) || - is_record_type(value->type)) - error_at("Invalid compound assignment operand", cur_token_loc()); - var_t *current = require_typed_var(parent, row->type); - current->var_name = gen_name(); - add_insn(parent, bb, OP_read, current, address, NULL, row->type->size, - NULL); - current = integer_promote_operand(parent, &bb, current); - value = integer_promote_operand(parent, &bb, value); - normalize_integer_binary_operands(parent, &bb, compound_op, ¤t, - &value); - var_t *updated = require_var(parent); - updated->var_name = gen_name(); - updated->type = integer_binary_result_type(compound_op, current, value); - add_insn(parent, bb, compound_op, updated, current, value, 0, NULL); - value = resize_to(parent, &bb, updated, row->type, 0); - } - value = convert_stored_value(parent, &bb, value, row->type, 0); - add_insn(parent, bb, OP_write, NULL, address, value, row->type->size, NULL); - lex_expect(T_semicolon); - return bb; -} - /* The middle operand of ?: is an expression, rather than merely an * assignment-expression, so top-level commas belong to the selected true * branch. Keep this small sequencing layer here until all full-expression entry @@ -4521,18 +3782,7 @@ static basic_block_t *handle_grouped_scalar_pointee_row_store(block_t *parent, */ void read_conditional_true_expression(block_t *parent, basic_block_t **bb) { - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - - while (lex_accept(T_comma)) { - discard_operand(parent, *bb); - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - } + read_comma_expression(parent, bb, false); } /* An integer constant expression with value zero is the sole scalar that can @@ -4545,7 +3795,7 @@ bool is_null_pointer_constant(var_t *value) !value->is_func && !value->init_val && !value->init_val_hi; } -void read_ternary_operation(block_t *parent, basic_block_t **bb) +static void read_ternary_operation_body(block_t *parent, basic_block_t **bb) { var_t *vd; @@ -4554,6 +3804,7 @@ void read_ternary_operation(block_t *parent, basic_block_t **bb) /* ternary-operator */ vd = opstack_pop(); + var_t *condition = vd; reject_record_operand(vd); add_insn(parent, *bb, OP_branch, NULL, vd, NULL, 0, NULL); @@ -4574,14 +3825,10 @@ void read_ternary_operation(block_t *parent, basic_block_t **bb) var_t *true_val = opstack_pop(); - /* false branch */ - if (!read_assignment_expression(parent, &else_)) - read_expr(parent, &else_); - - /* The third operand has conditional-expression grammar, making nested - * conditionals right-associative: a ? b : c ? d : e. + /* false branch The third operand has conditional-expression grammar, making + * nested conditionals right-associative: a ? b : c ? d : e. */ - read_ternary_operation(parent, &else_); + read_assignment_or_expression(parent, &else_); bb_connect(*bb, else_entry, ELSE); var_t *false_val = opstack_pop(); @@ -4638,15 +3885,13 @@ void read_ternary_operation(block_t *parent, basic_block_t **bb) */ true_addr = record_value_address(parent, then_, true_val); false_addr = record_value_address(parent, else_, false_val); - selected = require_ref_var(parent, true_val->type, 0); - selected->var_name = gen_name(); + selected = name_var(require_ref_var(parent, true_val->type, 0)); add_insn(parent, then_, OP_assign, selected, true_addr, NULL, 0, NULL); add_insn(parent, else_, OP_assign, selected, false_addr, NULL, 0, NULL); bb_connect(then_, end_ternary, NEXT); bb_connect(else_, end_ternary, NEXT); - vd = require_typed_var(parent, true_val->type); - vd->var_name = gen_name(); + vd = name_var(require_typed_var(parent, true_val->type)); add_insn(parent, end_ternary, OP_allocat, vd, NULL, NULL, 0, NULL); emit_record_copy_from_address(parent, &end_ternary, vd, selected); opstack_push(vd); @@ -4654,9 +3899,11 @@ void read_ternary_operation(block_t *parent, basic_block_t **bb) return; } - vd = require_var(parent); - vd->var_name = gen_name(); - if (true_pointee_signature || false_pointee_signature) { + vd = require_named_var(parent); + if (true_val->type == TY_void && !true_val->ptr_level && + false_val->type == TY_void && !false_val->ptr_level) { + vd->type = TY_void; + } else if (true_pointee_signature || false_pointee_signature) { func_t *signature = true_pointee_signature ? true_pointee_signature : false_pointee_signature; var_t *pointer_value = true_pointee_signature ? true_val : false_val; @@ -4734,11 +3981,36 @@ void read_ternary_operation(block_t *parent, basic_block_t **bb) vd->type = array_ref->type; } + if (unevaluated_expression_depth && condition->is_const && + !is_pointer_like_value(condition) && !is_pointer_like_value(vd) && + !is_record_type(vd->type) && vd->type != TY_void) { + var_t *selected = condition->init_val || condition->init_val_hi + ? true_val + : false_val; + vd->is_const = selected->is_const; + vd->init_val = selected->init_val; + vd->init_val_hi = selected->init_val_hi; + } vd->is_ternary_ret = true; opstack_push(vd); bb[0] = end_ternary; } +/* The third operand of `?:' re-enters the conditional grammar through + * read_assignment_or_expression(), whose read_expr() call has already returned + * by then, so a chain such as `x ? 1 : x ? 1 : ...' never raises expr_depth. + * Count each conditional against the same limit to diagnose a long chain before + * it exhausts the machine stack. + */ +void read_ternary_operation(block_t *parent, basic_block_t **bb) +{ + expr_depth++; + if (expr_depth > MAX_EXPR_DEPTH) + error_at("Expression nesting too deep", cur_token_loc()); + read_ternary_operation_body(parent, bb); + expr_depth--; +} + /* Lower the assignment whose left operand starts with the identifier ahead, * declared by @var. There is nothing to lower when @var is NULL. */ @@ -4757,13 +4029,11 @@ bool read_body_assignment(var_t *var, int one = 0; opcode_t op = OP_generic; lvalue_t lvalue; - int size = 0; /* has memory address that we want to set */ read_lvalue(&lvalue, var, parent, bb, false, OP_generic); while (lvalue_paren_depth-- > 0) lex_expect(T_close_bracket); - size = lvalue.size; if (lvalue.is_array && lvalue_write_follows(prefix_op)) error_at("assignment to expression with array type", @@ -4788,8 +4058,7 @@ bool read_body_assignment(var_t *var, */ if (lvalue.is_reference) { rs1 = opstack_pop(); - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); opstack_push(vd); add_insn(parent, *bb, OP_read, vd, rs1, NULL, PTR_SIZE, NULL); } @@ -4824,46 +4093,19 @@ bool read_body_assignment(var_t *var, * of stack as the one of operands and the destination. */ if (one == 1) { - if (lvalue.is_reference) { - t = opstack_pop(); - vd = require_var(parent); - vd->var_name = gen_name(); - vd->type = pointee_type_from_pointer_typedef(lvalue.type); - vd->ptr_level = lvalue.value_ptr_level; - opstack_push(vd); - if (is_bitfield(lvalue.decl)) { - opstack_pop(); - vd = read_bitfield_value(parent, bb, t, lvalue.decl); - opstack_push(vd); - } else { - add_insn(parent, *bb, OP_read, vd, t, NULL, lvalue.size, - NULL); - } - } else - t = operand_stack[operand_stack_idx - 1]; + t = read_lvalue_update_value(parent, bb, &lvalue); - vd = require_var(parent); - vd->var_name = gen_name(); - vd->init_val = increment_size; - add_insn(parent, *bb, OP_load_constant, vd, NULL, NULL, 0, - NULL); + vd = load_constant(parent, *bb, increment_size, TY_int); rs2 = vd; rs1 = opstack_pop(); - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); add_insn(parent, *bb, op, vd, rs1, rs2, 0, NULL); - if (lvalue.is_reference) { - if (is_bitfield(lvalue.decl)) - write_bitfield_value(parent, bb, t, vd, lvalue.decl); - else { - vd = convert_stored_value(parent, bb, vd, lvalue.type, - lvalue.value_ptr_level); - add_insn(parent, *bb, OP_write, NULL, t, vd, size, - NULL); - } - } else { + if (lvalue.is_reference) + store_addressed_assignment(parent, bb, t, &lvalue, vd, + false); + else { vd = resize_var(parent, bb, vd, t); mark_var_mutated(t); add_insn(parent, *bb, OP_assign, t, vd, NULL, 0, NULL); @@ -4871,48 +4113,18 @@ bool read_body_assignment(var_t *var, if (assignment_result) { if (!lvalue.is_reference) { assignment_result[0] = vd; - } else if (is_bitfield(lvalue.decl)) { - var_t *stored = reload_assigned_bitfield(parent, bb, t, - lvalue.decl); - assignment_result[0] = stored; } else { - var_t *stored = require_typed_var(parent, lvalue.type); - stored->ptr_level = lvalue.value_ptr_level; - stored->var_name = gen_name(); - stored->is_assignment_reload = true; - add_insn(parent, *bb, OP_read, stored, t, NULL, - lvalue.size, NULL); - assignment_result[0] = stored; + assignment_result[0] = read_addressed_value( + parent, bb, t, &lvalue, true, true); } } } else { - if (lvalue.is_reference) { - t = opstack_pop(); - vd = require_var(parent); - vd->var_name = gen_name(); - vd->type = pointee_type_from_pointer_typedef(lvalue.type); - vd->ptr_level = lvalue.value_ptr_level; - opstack_push(vd); - if (is_bitfield(lvalue.decl)) { - opstack_pop(); - vd = read_bitfield_value(parent, bb, t, lvalue.decl); - opstack_push(vd); - } else { - add_insn(parent, *bb, OP_read, vd, t, NULL, lvalue.size, - NULL); - } - } else - t = operand_stack[operand_stack_idx - 1]; + t = read_lvalue_update_value(parent, bb, &lvalue); - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - - /* read_expr stops before `?`; compound assignment has the - * same conditional-expression RHS grammar as ordinary - * assignment. - */ - read_ternary_operation(parent, bb); - } + /* Compound assignment has the same conditional-expression RHS + * grammar as ordinary assignment. + */ + read_assignment_or_expression(parent, bb); var_t *rhs_val = opstack_pop(); reject_record_operand(rhs_val); @@ -4929,23 +4141,10 @@ bool read_body_assignment(var_t *var, * add -56 to a wider left operand. */ rhs_val = integer_promote_operand(parent, bb, rhs_val); - opstack_push(rhs_val); - vd = require_var(parent); - vd->init_val = increment_size; - vd->var_name = gen_name(); - opstack_push(vd); - add_insn(parent, *bb, OP_load_constant, vd, NULL, NULL, 0, - NULL); - - rs2 = opstack_pop(); - rs1 = opstack_pop(); - vd = require_var(parent); - vd->var_name = gen_name(); - vd->type = integer_binary_result_type(OP_mul, rs1, rs2); - opstack_push(vd); - add_insn(parent, *bb, OP_mul, vd, rs1, rs2, 0, NULL); - - rs2 = opstack_pop(); + if (increment_size != 1) + rhs_val = scale_pointer_index(parent, *bb, rhs_val, + increment_size, true); + rs2 = rhs_val; rs1 = opstack_pop(); /* Compound assignment performs the same integer promotions and @@ -4961,27 +4160,14 @@ bool read_body_assignment(var_t *var, normalize_integer_binary_operands(parent, bb, op, &rs1, &rs2); } - vd = require_var(parent); - vd->var_name = gen_name(); - vd->type = integer_binary_result_type(op, rs1, rs2); - add_insn(parent, *bb, op, vd, rs1, rs2, 0, NULL); + vd = emit_binary_result( + parent, bb, op, rs1, rs2, + integer_binary_result_type(op, rs1, rs2)); - if (lvalue.is_reference) { - if (is_bitfield(lvalue.decl)) - write_bitfield_value(parent, bb, t, vd, lvalue.decl); - else { - /* Convert back to the object type before the store, as - * the direct-variable path does. The store alone - * narrows the bytes, but the expression value is this - * register, and a _Bool object must receive 0 or 1 - * rather than the low byte of the sum. - */ - vd = resize_to(parent, bb, vd, lvalue.type, - lvalue.value_ptr_level); - add_insn(parent, *bb, OP_write, NULL, t, vd, - lvalue.size, NULL); - } - } else { + if (lvalue.is_reference) + vd = store_addressed_assignment(parent, bb, t, &lvalue, vd, + true); + else { vd = resize_var(parent, bb, vd, t); add_insn(parent, *bb, OP_assign, t, vd, NULL, 0, NULL); } @@ -4991,20 +4177,15 @@ bool read_body_assignment(var_t *var, * the arithmetic result, so its expression value must be * reloaded just like an ordinary bit-field assignment. */ - if (is_bitfield(lvalue.decl)) { - var_t *stored = reload_assigned_bitfield(parent, bb, t, - lvalue.decl); - assignment_result[0] = stored; - } else { - assignment_result[0] = vd; - } + assignment_result[0] = + is_bitfield(lvalue.decl) + ? read_addressed_value(parent, bb, t, &lvalue, true, + true) + : vd; } } } else { - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } + read_assignment_or_expression(parent, bb); if (lvalue.is_func) { rs2 = opstack_pop(); @@ -5018,12 +4199,11 @@ bool read_body_assignment(var_t *var, * function address. */ if (rs2->is_func && find_var(rs2->var_name, parent) == rs2) { - t = require_ref_var(parent, rs2->type, rs2->ptr_level); - t->var_name = gen_name(); + t = name_var( + require_ref_var(parent, rs2->type, rs2->ptr_level)); add_insn(parent, *bb, OP_address_of, t, rs2, NULL, 0, NULL); - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); add_insn(parent, *bb, OP_read, vd, t, NULL, PTR_SIZE, NULL); rs2 = vd; } @@ -5032,9 +4212,8 @@ bool read_body_assignment(var_t *var, * variable. */ if (!lvalue.is_reference) { - var_t *addr = - require_ref_var(parent, lvalue.type, lvalue.ptr_level); - addr->var_name = gen_name(); + var_t *addr = name_var( + require_ref_var(parent, lvalue.type, lvalue.ptr_level)); add_insn(parent, *bb, OP_address_of, addr, rs1, NULL, 0, NULL); rs1 = addr; @@ -5078,15 +4257,11 @@ bool read_body_assignment(var_t *var, error_at("incompatible types in assignment", cur_token_loc()); - if (is_bitfield(lvalue.decl)) - write_bitfield_value(parent, bb, rs1, rs2, lvalue.decl); - else if (record_store) + if (record_store) emit_record_copy_to_address(parent, bb, rs1, rs2); - else { - rs2 = convert_stored_value(parent, bb, rs2, lvalue.type, - lvalue.value_ptr_level); - add_insn(parent, *bb, OP_write, NULL, rs1, rs2, size, NULL); - } + else + store_addressed_assignment(parent, bb, rs1, &lvalue, rs2, + false); if (assignment_result && record_store) { /* The stored record is a copy of the right operand. */ assignment_result[0] = rs2; @@ -5096,19 +4271,8 @@ bool read_body_assignment(var_t *var, * the source, so reload its extracted value instead of * leaking the unconverted right operand. */ - if (is_bitfield(lvalue.decl)) { - var_t *stored = reload_assigned_bitfield( - parent, bb, rs1, lvalue.decl); - assignment_result[0] = stored; - } else { - var_t *stored = require_typed_var(parent, lvalue.type); - stored->ptr_level = lvalue.value_ptr_level; - stored->var_name = gen_name(); - stored->is_assignment_reload = true; - add_insn(parent, *bb, OP_read, stored, rs1, NULL, size, - NULL); - assignment_result[0] = stored; - } + assignment_result[0] = read_addressed_value( + parent, bb, rs1, &lvalue, true, true); } } else { rs1 = opstack_pop(); @@ -5166,6 +4330,8 @@ bool assignment_expression_follows(void) int depth = 0; for (token_t *t = cur_token->next; t; t = t->next) { + if (!depth && assignment_operator(t->kind)) + return true; switch (t->kind) { case T_open_bracket: case T_open_square: @@ -5179,20 +4345,6 @@ bool assignment_expression_follows(void) return false; depth--; break; - case T_assign: - case T_pluseq: - case T_minuseq: - case T_asteriskeq: - case T_divideeq: - case T_modeq: - case T_lshifteq: - case T_rshifteq: - case T_andeq: - case T_oreq: - case T_xoreq: - if (!depth) - return true; - break; case T_question: /* `?:` has lower precedence than assignment. An assignment in an * arm belongs to that conditional expression, not to the leading @@ -5221,17 +4373,11 @@ bool assignment_expression_follows(void) bool compound_literal_starts_expression(void) { token_t *t = cur_token->next; - int depth = 0; if (!t || t->kind != T_open_bracket) return false; - for (; t; t = t->next) { - if (t->kind == T_open_bracket) - depth++; - else if (t->kind == T_close_bracket && --depth == 0) - return t->next && t->next->kind == T_open_curly; - } - return false; + t = skip_balanced_delimiter(t, T_open_bracket, T_close_bracket); + return t && t->next && t->next->kind == T_open_curly; } /* Whether the assignment ahead has a parenthesized left operand followed by @@ -5240,61 +4386,33 @@ bool compound_literal_starts_expression(void) * member chain on a dereference, `(*q).a.b = v`, stays with the dereference * path. */ +static bool postfix_lvalue_follows(const token_t *token) +{ + return token && (token->kind == T_dot || token->kind == T_arrow || + token->kind == T_open_square); +} + static bool grouped_postfix_lvalue_follows(void) { token_t *token = cur_token->next; token_t *inner; - int depth = 0; if (!token || token->kind != T_open_bracket) return false; inner = token->next; - for (; token; token = token->next) { - if (token->kind == T_open_bracket) - depth++; - else if (token->kind == T_close_bracket && !--depth) - break; - } - if (!token || !(token = token->next) || - (token->kind != T_dot && token->kind != T_arrow && - token->kind != T_open_square)) + token = skip_balanced_delimiter(token, T_open_bracket, T_close_bracket); + if (!token || !(token = token->next) || !postfix_lvalue_follows(token)) return false; if (inner && inner->kind == T_asterisk) { while (token && token->kind == T_dot && token->next && token->next->kind == T_identifier) token = token->next->next; - if (token && token->kind != T_dot && token->kind != T_arrow && - token->kind != T_open_square) + if (!postfix_lvalue_follows(token)) return false; } return true; } -/* Whether the next token assigns to what precedes it. */ -static bool assignment_operator_follows(void) -{ - token_t *token = cur_token->next; - - if (!token) - return false; - switch (token->kind) { - case T_assign: - case T_pluseq: - case T_minuseq: - case T_asteriskeq: - case T_divideeq: - case T_modeq: - case T_lshifteq: - case T_rshifteq: - case T_xoreq: - case T_oreq: - case T_andeq: - return true; - default: - return false; - } -} - /* Whether the assignment ahead has a call result as the root of its left * operand, `one()->y = v` or `rows()[0][1].x = v`. No declaration describes * such an operand, so the lvalue parser cannot follow it, but the postfix @@ -5303,23 +4421,16 @@ static bool assignment_operator_follows(void) static bool call_postfix_lvalue_follows(void) { token_t *token = cur_token->next; - int depth = 0; if (!token || token->kind != T_identifier) return false; token = token->next; if (!token || token->kind != T_open_bracket) return false; - for (; token; token = token->next) { - if (token->kind == T_open_bracket) - depth++; - else if (token->kind == T_close_bracket && !--depth) - break; - } + token = skip_balanced_delimiter(token, T_open_bracket, T_close_bracket); if (!token || !(token = token->next)) return false; - return token->kind == T_dot || token->kind == T_arrow || - token->kind == T_open_square; + return postfix_lvalue_follows(token); } /* Assign through a left operand that the expression parser lowers as a value, @@ -5349,17 +4460,16 @@ static void read_grouped_postfix_assignment(block_t *parent, basic_block_t **bb) target->is_func) error_at("Assignment requires a modifiable lvalue", cur_token_loc()); pointer_object = effective_pointer_depth(target) > 0; - if (pointer_object ? target->is_const_pointer : target->is_const_qualified) + if ((pointer_object && target->is_const_pointer) || + (!pointer_object && target->is_const_qualified) || + (target->func_signature && target->is_const_qualified)) error_at("assignment of read-only location", cur_token_loc()); address = target->compound_literal_address; size = pointer_object || target->func_signature ? PTR_SIZE : target->type->size; - if (!lex_accept(T_assign) && !accept_compound_assign_op(&compound_op)) + if (!accept_assignment_op(&compound_op)) error_at("Expected assignment operator", cur_token_loc()); - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } + read_assignment_or_expression(parent, bb); value = opstack_pop(); if (is_record_object(target)) { @@ -5379,54 +4489,19 @@ static void read_grouped_postfix_assignment(block_t *parent, basic_block_t **bb) (pointer_object && compound_op != OP_add && compound_op != OP_sub)))) error_at("Invalid assignment operand", cur_token_loc()); - if (compound_op != OP_generic) { - var_t *current; - var_t *combined; - - if (target->compound_literal_bitfield) - current = read_bitfield_value(parent, bb, address, - target->compound_literal_bitfield); - else { - current = - require_typed_ptr_var(parent, target->type, target->ptr_level); - current->var_name = gen_name(); - add_insn(parent, *bb, OP_read, current, address, NULL, size, NULL); - } - if (is_pointer_operation(compound_op, current, value)) { - handle_pointer_arithmetic(parent, bb, compound_op, current, value); - combined = opstack_pop(); - } else { - current = integer_promote_operand(parent, bb, current); - value = integer_promote_operand(parent, bb, value); - normalize_integer_binary_operands(parent, bb, compound_op, ¤t, - &value); - combined = require_var(parent); - combined->var_name = gen_name(); - combined->type = - integer_binary_result_type(compound_op, current, value); - add_insn(parent, *bb, compound_op, combined, current, value, 0, - NULL); - } - value = combined; - } - - if (target->compound_literal_bitfield) { - write_bitfield_value(parent, bb, address, value, - target->compound_literal_bitfield); - result = reload_assigned_bitfield(parent, bb, address, - target->compound_literal_bitfield); - opstack_push(result); - return; - } - if (!value->is_func) - value = resize_to(parent, bb, value, target->type, target->ptr_level); - add_insn(parent, *bb, OP_write, NULL, address, value, size, NULL); - - /* The assignment has the value the object then holds. */ - result = require_typed_ptr_var(parent, target->type, target->ptr_level); - result->var_name = gen_name(); + lvalue_t target_lvalue = { + .size = size, + .value_ptr_level = target->ptr_level, + .type = target->type, + .decl = target->compound_literal_bitfield, + }; + value = combine_addressed_assignment(parent, bb, compound_op, address, + &target_lvalue, value); + store_addressed_assignment(parent, bb, address, &target_lvalue, value, + true); + result = + read_addressed_value(parent, bb, address, &target_lvalue, true, false); result->func_signature = target->func_signature; - add_insn(parent, *bb, OP_read, result, address, NULL, size, NULL); opstack_push(result); } @@ -5453,22 +4528,9 @@ static var_t *dereferenced_array_store_base(block_t *parent, int *stars) if (!array || !is_array_declarator(array) || array->is_func || array->func_signature || array->pointee_func_signature) return NULL; - switch (token->next->kind) { - case T_assign: - case T_pluseq: - case T_minuseq: - case T_asteriskeq: - case T_divideeq: - case T_modeq: - case T_lshifteq: - case T_rshifteq: - case T_xoreq: - case T_oreq: - case T_andeq: - return array; - default: + if (!assignment_operator(token->next->kind)) return NULL; - } + return array; } bool read_assignment_expression(block_t *parent, basic_block_t **bb) @@ -5610,75 +4672,38 @@ bool read_assignment_expression(block_t *parent, basic_block_t **bb) (address_mask & (1U << (address_depth - 2)))) || (!field && points_to_const_callback(object_address))) error_at("assignment of read-only location", cur_token_loc()); - if (!lex_accept(T_assign) && !accept_compound_assign_op(&compound_op)) + if (!accept_assignment_op(&compound_op)) error_at("Expected assignment after pointer dereference", cur_token_loc()); - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } + read_assignment_or_expression(parent, bb); value = opstack_pop(); - if (compound_op != OP_generic) { - var_t *current; - if (field && is_bitfield(field)) { - current = read_bitfield_value(parent, bb, address, field); - } else { - current = require_typed_var(parent, value_type); - current->ptr_level = value_ptr_level; - current->var_name = gen_name(); - add_insn(parent, *bb, OP_read, current, address, NULL, - store_size, NULL); - } - /* A pointer object steps by its element size. */ - if (is_pointer_operation(compound_op, current, value)) { - handle_pointer_arithmetic(parent, bb, compound_op, current, - value); - value = opstack_pop(); - } else { - current = integer_promote_operand(parent, bb, current); - value = integer_promote_operand(parent, bb, value); - normalize_integer_binary_operands(parent, bb, compound_op, - ¤t, &value); - var_t *combined = require_var(parent); - combined->var_name = gen_name(); - combined->type = - integer_binary_result_type(compound_op, current, value); - add_insn(parent, *bb, compound_op, combined, current, value, 0, - NULL); - value = combined; - } - } - - if (field && is_bitfield(field)) - write_bitfield_value(parent, bb, address, value, field); - else if (is_record_type(value_type) && !value_ptr_level) { + lvalue_t target = { + .size = store_size, + .value_ptr_level = value_ptr_level, + .type = value_type, + .decl = field, + }; + value = combine_addressed_assignment(parent, bb, compound_op, address, + &target, value); + + if (!is_bitfield(field) && is_record_type(value_type) && + !value_ptr_level) { if (compound_op != OP_generic || !is_record_object(value)) error_at("Invalid record assignment", cur_token_loc()); emit_record_copy_to_address(parent, bb, address, value); opstack_push(value); return true; } else { - value = convert_stored_value(parent, bb, value, value_type, - value_ptr_level); - add_insn(parent, *bb, OP_write, NULL, address, value, store_size, - NULL); + store_addressed_assignment(parent, bb, address, &target, value, + false); } /* Reload after the store: assignment expressions observe the stored * object representation, including narrow integer conversion. */ - if (field && is_bitfield(field)) { - result = reload_assigned_bitfield(parent, bb, address, field); - } else { - result = require_typed_var(parent, value_type); - result->ptr_level = value_ptr_level; - result->var_name = gen_name(); - result->is_assignment_reload = true; - add_insn(parent, *bb, OP_read, result, address, NULL, store_size, - NULL); - } + result = read_addressed_value(parent, bb, address, &target, true, true); if (is_bool_scalar(result->type, result->ptr_level)) result = normalize_bool(parent, bb, result); opstack_push(result); diff --git a/src/parser-global.c b/src/parser-global.c index 3580ed99..db6b1abf 100644 --- a/src/parser-global.c +++ b/src/parser-global.c @@ -106,6 +106,20 @@ static bool identifier_list_param_matches(const var_t *defined, return compatible_function_param_decl(defined, declared); } +static void error_named_declaration(const char *message, const var_t *var) +{ + char diagnostic[MAX_LINE_LEN]; + + snprintf(diagnostic, sizeof(diagnostic), "%s '%s'", message, + file_scope_var_name(var)); + error_at(diagnostic, next_token_loc()); +} + +static void error_conflicting_function_declaration(const var_t *var) +{ + error_named_declaration("conflicting types for function declaration", var); +} + /* A declarator's base type is already known when this runs. Keeping function * completion independent of how that type was spelled lets enum, record, and * ordinary scalar declarations share linkage and redeclaration checks. @@ -121,12 +135,30 @@ bool read_global_function_declarator(block_t *block, * redeclaration. Without this check the back end emitted colliding labels * and the resulting program could jump through object storage. */ - var_t *object = find_global_var(var->var_name); - + const char *source_name = file_scope_var_name(var); + var_t *unit_decl = find_tu_declaration(source_name); + var_t *object = + unit_decl && !find_func(unit_decl->var_name) + ? unit_decl + : (!is_static && !unit_decl ? find_global_var(source_name) : NULL); + + if (find_block_ordinary(CURRENT_TU_SCOPE, var->var_name, + ORDINARY_CONSTANT | ORDINARY_TYPEDEF, NULL)) + error_at("function declaration conflicts with an ordinary identifier", + next_token_loc()); if (object && object != var) error_at("function declaration conflicts with global object", next_token_loc()); + if (unit_decl && find_func(unit_decl->var_name) && unit_decl->is_static) { + var->source_name = unit_decl->source_name; + var->var_name = unit_decl->var_name; + } else if (is_static && !unit_decl) { + assign_internal_linkage_name(var, source_name); + } + + add_tu_global_var(var); + func_t *func = find_func(var->var_name); func_t func_tmp; bool check_decl = false; @@ -169,7 +201,6 @@ bool read_global_function_declarator(block_t *block, next_token_loc()); func->is_static = check_decl && func_tmp.is_static ? true : is_static; func->is_inline = var->is_inline; - var_reset_subscripts(&func->return_def); block->locals.size--; /* Parse this declarator independently of any earlier declaration. A later @@ -203,19 +234,16 @@ bool read_global_function_declarator(block_t *block, func->return_def.ptr_level != func_tmp.return_def.ptr_level || func->return_def.is_const_qualified != func_tmp.return_def.is_const_qualified) - error_at("conflicting types for function declaration", - next_token_loc()); + error_conflicting_function_declaration(var); if (func->has_prototype && func_tmp.has_prototype) { if (func->num_params != func_tmp.num_params || func->va_args != func_tmp.va_args) - error_at("conflicting types for function declaration", - next_token_loc()); + error_conflicting_function_declaration(var); for (int i = 0; i < func->num_params; i++) { const var_t *now = &func->param_defs[i]; const var_t *before = &func_tmp.param_defs[i]; if (!compatible_function_param_decl(now, before)) - error_at("conflicting types for function declaration", - next_token_loc()); + error_conflicting_function_declaration(var); } } else if (func->has_prototype && !func_tmp.has_prototype && func_tmp.num_params) { @@ -223,13 +251,11 @@ bool read_global_function_declarator(block_t *block, * promoted parameter types. */ if (func->num_params != func_tmp.num_params || func->va_args) - error_at("conflicting types for function declaration", - next_token_loc()); + error_conflicting_function_declaration(var); for (int i = 0; i < func->num_params; i++) if (!identifier_list_param_matches(&func_tmp.param_defs[i], &func->param_defs[i])) - error_at("conflicting types for function declaration", - next_token_loc()); + error_conflicting_function_declaration(var); } else if (strict_c99 && func->has_prototype && !func_tmp.has_prototype) { /* A variadic prototype and parameters promoted from char, short, or @@ -238,26 +264,22 @@ bool read_global_function_declarator(block_t *block, * mode, where existing old-style sources rely on it. */ if (func->va_args) - error_at("conflicting types for function declaration", - next_token_loc()); + error_conflicting_function_declaration(var); for (int i = 0; i < func->num_params; i++) if (parameter_changes_under_default_promotion( &func->param_defs[i])) - error_at("conflicting types for function declaration", - next_token_loc()); + error_conflicting_function_declaration(var); } else if (!func->has_prototype && func->num_params && func_tmp.has_prototype) { /* An old-style definition of a function declared with a prototype * keeps its own parameter names and that prototype. */ if (func->num_params != func_tmp.num_params || func_tmp.va_args) - error_at("conflicting types for function declaration", - next_token_loc()); + error_conflicting_function_declaration(var); for (int i = 0; i < func->num_params; i++) if (!identifier_list_param_matches(&func->param_defs[i], &func_tmp.param_defs[i])) - error_at("conflicting types for function declaration", - next_token_loc()); + error_conflicting_function_declaration(var); func->has_prototype = true; } else if (!func->has_prototype && func_tmp.has_prototype) { /* An empty-list definition has no named parameters. It cannot @@ -267,8 +289,7 @@ bool read_global_function_declarator(block_t *block, */ if (lex_peek(T_open_curly, NULL) && (func_tmp.num_params || func_tmp.va_args)) - error_at("conflicting types for function declaration", - next_token_loc()); + error_conflicting_function_declaration(var); /* A prior prototype remains visible after a compatible `f()` * declaration and still constrains subsequent calls. @@ -296,7 +317,7 @@ bool read_global_function_declarator(block_t *block, error_at("function definition cannot take its type from a typedef", next_token_loc()); if (check_decl && func_tmp.bbs) - error_at("redefinition of function", next_token_loc()); + error_named_declaration("redefinition of function", var); if (is_incomplete_record_object(&func->return_def)) error_at("function definition cannot return incomplete record type", next_token_loc()); @@ -334,19 +355,35 @@ var_t *resolve_global_declarator(block_t *block, bool is_static, bool *is_redeclaration) { + const char *source_name = file_scope_var_name(var); + var_t *unit_decl = find_tu_declaration(source_name); var_t *previous = NULL; *is_redeclaration = false; + if (find_block_ordinary(CURRENT_TU_SCOPE, var->var_name, + ORDINARY_CONSTANT | ORDINARY_TYPEDEF, NULL)) + error_at( + "global object declaration conflicts with an ordinary identifier", + next_token_loc()); + /* The ordinary identifier namespace is shared with functions. This is * intentionally before object redeclaration handling: a function is not a * compatible tentative definition of an object, even when both happen to * have the same declared scalar type. */ - if (find_func(var->var_name)) + if ((unit_decl && find_func(unit_decl->var_name)) || + (!is_static && !unit_decl && find_func(var->var_name))) error_at("global object declaration conflicts with function", next_token_loc()); + if (unit_decl && unit_decl->is_static) { + var->source_name = unit_decl->source_name; + var->var_name = unit_decl->var_name; + } else if (is_static && !unit_decl) { + assign_internal_linkage_name(var, source_name); + } + for (int i = 0; i + 1 < block->locals.size; i++) { var_t *candidate = block->locals.elements[i]; if (!strcmp(candidate->var_name, var->var_name)) { @@ -354,8 +391,10 @@ var_t *resolve_global_declarator(block_t *block, break; } } - if (!previous) + if (!previous) { + add_tu_global_var(var); return var; + } *is_redeclaration = true; @@ -386,12 +425,13 @@ var_t *resolve_global_declarator(block_t *block, previous->pointee_array_dim4 != var->pointee_array_dim4 || previous->is_const_qualified != var->is_const_qualified || previous->is_volatile != var->is_volatile) - error_at("conflicting types for global declaration", next_token_loc()); + error_named_declaration("conflicting types for global declaration", + var); if (!previous->is_static && is_static) error_at("static declaration follows non-static declaration", next_token_loc()); if (lex_peek(T_assign, NULL) && previous->has_initializer) - error_at("redefinition of global variable", next_token_loc()); + error_named_declaration("redefinition of global variable", var); /* A later bound completes the shared object's type. */ if (previous->has_unsized_array && !var->has_unsized_array) { @@ -406,9 +446,53 @@ var_t *resolve_global_declarator(block_t *block, if (operand_stack_idx && operand_stack[operand_stack_idx - 1] == var) operand_stack[operand_stack_idx - 1] = previous; block->locals.size--; + add_tu_global_var(previous); return previous; } +static var_t *prepare_global_object_definition(block_t *block, + var_t *var, + bool is_static) +{ + bool is_redeclaration; + bool is_definition = !var->is_extern; + + if (is_definition && is_incomplete_record_object(var)) + error_at("Incomplete struct/union type cannot define an object", + cur_token_loc()); + var = resolve_global_declarator(block, var, is_static, &is_redeclaration); + if (is_definition && (!is_redeclaration || var->is_extern)) { + var->is_extern = false; + add_insn(block, GLOBAL_FUNC->bbs, OP_allocat, var, NULL, NULL, 0, NULL); + } + return var; +} + +static var_t *read_global_declarator_start(block_t *block, + type_t *decl_type, + bool is_const, + bool is_static, + bool is_volatile, + bool is_extern, + bool is_inline, + bool full) +{ + var_t *var = + full ? require_var(block) : require_typed_var(block, decl_type); + + var->is_global = true; + var->is_static = is_static; + var->is_inline = is_inline; + var->is_const_qualified = is_const; + var->is_volatile = is_volatile; + if (full) + read_full_var_decl(var, false, false, false); + else + read_inner_var_decl(var, false, false, false); + var->is_extern = is_extern && !lex_peek(T_assign, NULL); + return var; +} + /* Read one declarator after the first in a global declaration. Each shares the * declaration's base type: "int a = 1, b, c = 3;". */ @@ -420,34 +504,19 @@ bool read_global_declarator(block_t *block, bool is_extern, bool allow_definition) { - bool is_redeclaration; - var_t *nv = require_typed_var(block, decl_type); - nv->is_global = true; - nv->is_static = is_static; - nv->is_const_qualified = is_const; - nv->is_volatile = is_volatile; - read_inner_var_decl(nv, false, false, false); - nv->is_extern = is_extern && !lex_peek(T_assign, NULL); + var_t *nv = + read_global_declarator_start(block, decl_type, is_const, is_static, + is_volatile, is_extern, false, false); if (lex_peek(T_open_bracket, NULL) || (nv->is_func && nv->type->is_direct_function_type)) return read_global_function_declarator(block, nv, is_static, allow_definition); - bool is_definition = !nv->is_extern; - if (is_definition && is_incomplete_record_object(nv)) - error_at("Incomplete struct/union type cannot define an object", - cur_token_loc()); - nv = resolve_global_declarator(block, nv, is_static, &is_redeclaration); - if (is_definition && (!is_redeclaration || nv->is_extern)) { - nv->is_extern = false; - add_insn(block, GLOBAL_FUNC->bbs, OP_allocat, nv, NULL, NULL, 0, NULL); - } + nv = prepare_global_object_definition(block, nv, is_static); read_global_init_var(nv, block); discard_global_declarator_operand(nv); return false; } -void consume_global_compound_literal(void); - /* Lower a scalar record initializer into the global initializer block. Global * array elements already use parse_struct_field_init(); scalar records need the * same field-address writes rather than merely consuming their braces. @@ -483,7 +552,7 @@ void parse_global_compound_record_init(var_t *var, block_t *block) compound_type = record_kind ? find_record_tag(type_name, var->scope, record_kind) - : find_type(type_name, 1); + : find_visible_type(type_name, var->scope); target_type = var->type; if (target_type->base_type == TYPE_typedef && target_type->base_struct) target_type = target_type->base_struct; @@ -540,6 +609,8 @@ void parse_global_compound_array_init(var_t *var, block_t *block) type_t *element_type; var_t *array; int element_ptr_level = 0; + paren_type_name_t shape = {0}; + bool parenthesized_array; /* The element type may be qualified or spelled with several keywords, as in * `(const unsigned char[])`. @@ -549,148 +620,65 @@ void parse_global_compound_array_init(var_t *var, block_t *block) read_type_name_specifiers(var->scope ? var->scope : GLOBAL_BLOCK); while (lex_accept(T_asterisk)) { element_ptr_level++; - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; + skip_type_qualifiers(); } - /* `(int (*[])[2]){...}` is an array whose elements are pointers to rows. - * The inner suffix supplies the backing array bound; the suffixes after `)` - * describe each pointer element's pointee. - */ - if (lex_accept(T_open_bracket)) { - int pointee_dims = 0; - - if (element_ptr_level || !lex_peek(T_asterisk, NULL)) - error_at("Array compound literal needs a pointer declarator", - cur_token_loc()); - do { - lex_expect(T_asterisk); - element_ptr_level++; - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; - } while (lex_peek(T_asterisk, NULL)); - lex_expect(T_open_square); - - array = require_typed_var(GLOBAL_BLOCK, element_type); - array->var_name = gen_name(); - array->is_global = true; - array->ptr_level = element_ptr_level; - if (!lex_peek(T_close_square, NULL)) { - array->array_size = read_const_expr(GLOBAL_BLOCK); - if (array->array_size <= 0) - error_at("Array compound literal needs a positive bound", - cur_token_loc()); - } else { - array->has_unsized_array = true; - } - lex_expect(T_close_square); - lex_expect(T_close_bracket); - while (lex_accept(T_open_square)) { - int bound = read_const_expr(GLOBAL_BLOCK); - - if (pointee_dims >= 4) - error_at("Array declarators support at most four dimensions", - cur_token_loc()); - if (bound <= 0) - error_at("Array size must be positive", cur_token_loc()); - if (pointee_dims == 0) - array->pointee_array_size = bound; - else { - if (pointee_dims == 1) - array->pointee_array_dim2 = bound; - else if (pointee_dims == 2) - array->pointee_array_dim3 = bound; - else - array->pointee_array_dim4 = bound; - array->pointee_array_size *= bound; - } - lex_expect(T_close_square); - pointee_dims++; - } + parenthesized_array = read_compound_array_shape( + GLOBAL_BLOCK, element_ptr_level, true, &shape) && + shape.parenthesized_array; + array = name_var(require_typed_var(GLOBAL_BLOCK, element_type)); + array->is_global = true; + array->ptr_level = + parenthesized_array ? shape.array_element_ptr_level : element_ptr_level; + array->array_size = shape.array_size; + array->array_dim2 = shape.array_dim2; + array->array_dim3 = shape.array_dim3; + array->array_dim4 = shape.array_dim4; + array->has_unsized_array = shape.array_outer_unsized; + if (parenthesized_array) { + array->pointee_array_size = shape.pointee_array_size; + array->pointee_array_dim2 = shape.pointee_array_dim2; + array->pointee_array_dim3 = shape.pointee_array_dim3; + array->pointee_array_dim4 = shape.pointee_array_dim4; lex_expect(T_close_bracket); if (!element_type || element_type != var->type || - var->ptr_level != element_ptr_level + 1 || + var->ptr_level != array->ptr_level + 1 || var->pointee_array_size != array->pointee_array_size || var->pointee_array_dim2 != array->pointee_array_dim2 || var->pointee_array_dim3 != array->pointee_array_dim3 || var->pointee_array_dim4 != array->pointee_array_dim4) error_at("Incompatible array compound literal", cur_token_loc()); - if (!lex_peek(T_open_curly, NULL)) - error_at("Array compound literal needs an initializer", - next_token_loc()); - add_insn(GLOBAL_BLOCK, GLOBAL_FUNC->bbs, OP_allocat, array, NULL, NULL, - 0, NULL); - parse_array_init(array, GLOBAL_BLOCK, &GLOBAL_FUNC->bbs); - add_insn(block, GLOBAL_FUNC->bbs, OP_assign, var, array, NULL, 0, NULL); - return; - } - array = require_typed_var(GLOBAL_BLOCK, element_type); - array->var_name = gen_name(); - array->is_global = true; - array->ptr_level = element_ptr_level; - for (int dim = 0; lex_accept(T_open_square); dim++) { - int bound; + } else { + lex_expect(T_close_bracket); - if (dim >= 4) - error_at("Array compound literal supports at most four dimensions", - cur_token_loc()); - if (lex_peek(T_close_square, NULL)) { - if (dim) - error_at("Only the outer array bound may be inferred", - cur_token_loc()); - array->has_unsized_array = true; - lex_expect(T_close_square); - continue; - } - bound = read_const_expr(GLOBAL_BLOCK); - if (bound <= 0) - error_at("Array compound literal needs a positive bound", - cur_token_loc()); - if (!dim) - array->array_size = bound; - else { - if (dim == 1) - array->array_dim2 = bound; - else if (dim == 2) - array->array_dim3 = bound; - else - array->array_dim4 = bound; - array->array_size *= bound; + /* The backing array has an outer compound-literal bound and the typedef + * element's inner row shape. parse_array_init() keeps that shape on + * var_t rather than type_t. + */ + if (element_type && element_type->array_size) { + int trailing = fixed_array_trailing_count(element_type->array_dim2, + element_type->array_dim3, + element_type->array_dim4); + if (element_type->array_dim4) + error_at( + "Array compound literal supports at most four dimensions", + cur_token_loc()); + array->array_dim2 = element_type->array_size / trailing; + array->array_dim3 = element_type->array_dim2; + array->array_dim4 = element_type->array_dim3; + if (array->array_size) { + reject_oversized_array(array->array_size, + element_type->array_size); + array->array_size *= element_type->array_size; + } } - lex_expect(T_close_square); - } - lex_expect(T_close_bracket); - /* The backing array has an outer compound-literal bound and the typedef - * element's inner row shape. parse_array_init() keeps that shape on var_t - * rather than type_t. - */ - if (element_type && element_type->array_size) { - int trailing = 1; - - if (element_type->array_dim2) - trailing *= element_type->array_dim2; - if (element_type->array_dim3) - trailing *= element_type->array_dim3; - if (element_type->array_dim4) - trailing *= element_type->array_dim4; - if (element_type->array_dim4) - error_at("Array compound literal supports at most four dimensions", - cur_token_loc()); - array->array_dim2 = element_type->array_size / trailing; - array->array_dim3 = element_type->array_dim2; - array->array_dim4 = element_type->array_dim3; - if (array->array_size) - array->array_size *= element_type->array_size; + if (!element_type || element_type != var->type || + element_ptr_level + 1 != var->ptr_level || + var->pointee_array_size != array->array_dim2) + error_at("Incompatible array compound literal", cur_token_loc()); } - - if (!element_type || element_type != var->type || - element_ptr_level + 1 != var->ptr_level || - var->pointee_array_size != array->array_dim2) - error_at("Incompatible array compound literal", cur_token_loc()); if (!lex_peek(T_open_curly, NULL)) error_at("Array compound literal needs an initializer", next_token_loc()); @@ -701,64 +689,6 @@ void parse_global_compound_array_init(var_t *var, block_t *block) add_insn(block, GLOBAL_FUNC->bbs, OP_assign, var, array, NULL, 0, NULL); } -/* Struct and union objects accept brace initializers, unlike scalar globals. - * Keep their continuation declarators on the same path as the first one so that - * linkage, qualifiers, and declarator-specific modifiers cannot diverge. - */ -bool read_global_record_declarator(block_t *block, - type_t *decl_type, - bool is_const, - bool is_static, - bool is_volatile, - bool is_extern, - bool allow_definition) -{ - bool is_redeclaration; - var_t *var = require_typed_var(block, decl_type); - var->is_global = true; - var->is_static = is_static; - var->is_const_qualified = is_const; - var->is_volatile = is_volatile; - read_inner_var_decl(var, false, false, false); - var->is_extern = is_extern && !lex_peek(T_assign, NULL); - if (lex_peek(T_open_bracket, NULL) || - (var->is_func && var->type->is_direct_function_type)) - return read_global_function_declarator(block, var, is_static, - allow_definition); - bool is_definition = !var->is_extern; - if (is_definition && is_incomplete_record_object(var)) - error_at("Incomplete struct/union type cannot define an object", - cur_token_loc()); - var = resolve_global_declarator(block, var, is_static, &is_redeclaration); - if (is_definition && (!is_redeclaration || var->is_extern)) { - var->is_extern = false; - add_insn(block, GLOBAL_FUNC->bbs, OP_allocat, var, NULL, NULL, 0, NULL); - } - - if (!lex_accept(T_assign)) { - discard_global_declarator_operand(var); - return false; - } - - var->has_initializer = true; - - if (lex_peek(T_open_curly, NULL) && - (var->array_size > 0 || var->has_unsized_array || var->ptr_level > 0)) { - parse_array_init(var, block, &GLOBAL_FUNC->bbs); - } else if (global_compound_literal_starts_here() && - (var->ptr_level || var->type->ptr_level)) { - parse_global_compound_array_init(var, block); - } else if (global_compound_literal_starts_here()) { - parse_global_compound_record_init(var, block); - } else if (lex_peek(T_open_curly, NULL)) { - parse_global_record_init(var, block); - } else { - read_global_assignment_var(var); - } - discard_global_declarator_operand(var); - return false; -} - /* Read the declarators of a file-scope declaration through its terminating * semicolon, or through the body of a function definition. Every declarator * shares the base type, and object and function declarators may be mixed, as in @@ -771,22 +701,14 @@ void read_global_declarator_list(block_t *block, bool is_const, bool is_static, bool is_volatile, - bool is_extern, - bool is_record) + bool is_extern) { bool first = true; read_type_qualifiers(&is_const, &is_volatile, false); do { - bool ended = - is_record - ? read_global_record_declarator(block, decl_type, is_const, - is_static, is_volatile, - is_extern, first) - : read_global_declarator(block, decl_type, is_const, is_static, - is_volatile, is_extern, first); - - if (ended) + if (read_global_declarator(block, decl_type, is_const, is_static, + is_volatile, is_extern, first)) return; first = false; } while (lex_accept(T_comma)); @@ -800,17 +722,9 @@ void read_global_decl(block_t *block, bool is_inline, bool is_volatile) { - bool is_redeclaration; - var_t *var = require_var(block); - var->is_global = true; - var->is_static = is_static; - var->is_inline = is_inline; - var->is_const_qualified = is_const; - var->is_volatile = is_volatile; - - /* new function, or variables under parent */ - read_full_var_decl(var, false, false, false); - var->is_extern = is_extern && !lex_peek(T_assign, NULL); + var_t *var = + read_global_declarator_start(block, NULL, is_const, is_static, + is_volatile, is_extern, is_inline, true); /* `unary_t f;` with a function typedef declares the function f. */ if (lex_peek(T_open_bracket, NULL) || @@ -818,20 +732,10 @@ void read_global_decl(block_t *block, if (read_global_function_declarator(block, var, is_static, true)) return; } else { - bool is_definition = !var->is_extern; if (var->is_inline) error_at("inline specifier requires a function declarator", next_token_loc()); - if (is_definition && is_incomplete_record_object(var)) - error_at("Incomplete struct/union type cannot define an object", - cur_token_loc()); - var = - resolve_global_declarator(block, var, is_static, &is_redeclaration); - if (is_definition && (!is_redeclaration || var->is_extern)) { - var->is_extern = false; - add_insn(block, GLOBAL_FUNC->bbs, OP_allocat, var, NULL, NULL, 0, - NULL); - } + var = prepare_global_object_definition(block, var, is_static); /* is a variable */ if (lex_peek(T_assign, NULL)) { @@ -856,56 +760,9 @@ void read_global_decl(block_t *block, lex_expect(T_semicolon); } -void consume_global_compound_literal(void) -{ - lex_expect(T_open_curly); - - if (!lex_peek(T_close_curly, NULL)) { - for (;;) { - /* Just consume constant values for now */ - if (lex_peek(T_numeric, NULL)) { - lex_accept(T_numeric); - } else if (lex_peek(T_minus, NULL)) { - lex_accept(T_minus); - lex_accept(T_numeric); - } else if (lex_peek(T_string, NULL)) { - lex_accept(T_string); - } else if (lex_peek(T_char, NULL) || lex_peek(T_wchar, NULL)) { - lex_next(); - } else { - error_at( - "Global struct initialization requires constant values", - next_token_loc()); - } - - if (!lex_accept(T_comma)) - break; - if (lex_peek(T_close_curly, NULL)) - break; - } - } - lex_expect(T_close_curly); -} - void initialize_struct_field(var_t *nv, var_t *v, int offset) { - nv->type = v->type; - nv->var_name = ""; - nv->ptr_level = 0; - nv->is_func = false; - nv->is_global = false; - nv->is_const_qualified = false; - nv->array_size = 0; - nv->offset = offset; - nv->is_bitfield = false; - nv->bit_width = 0; - nv->bit_offset = 0; - nv->bit_storage_size = 0; - nv->init_val = 0; - nv->base = NULL; - nv->subscript = 0; - var_reset_subscripts(nv); - nv->is_compound_literal = false; + *nv = (var_t) {.type = v->type, .var_name = "", .offset = offset}; } /* The scalar or typedef-name specifier of a file-scope typedef, and any @@ -923,7 +780,7 @@ static const type_t *read_global_typedef_base(bool *typedef_const, if (!base) { lex_ident(T_identifier, base_type); - base = find_type(base_type, true); + base = find_visible_type(base_type, CURRENT_TU_SCOPE); } if (!base) error_at("Unable to find base type", cur_token_loc()); @@ -939,288 +796,84 @@ static void read_global_typedef_declarator(block_t *block, bool typedef_volatile, const type_t *base) { - type_t *type = add_type(); - - type->base_type = base->base_type; - type->size = base->size; - - /* A typedef of a record typedef remains a record type. Sharing its - * immutable member table preserves ordinary `alias.member` and - * `pointer_alias->member` lookup instead of turning the alias into a scalar - * descriptor with no fields. - */ - type->fields = base->fields; - type->num_fields = base->num_fields; - type->base_struct = base->base_struct; - if (base->base_type == TYPE_typedef && base->num_fields && - !type->base_struct) - type->base_struct = (type_t *) base; - type->alignment = base->alignment; - type->is_union = base->is_union; - type->has_flexible_array_member = base->has_flexible_array_member; - type->ptr_level = base->ptr_level; - type->pointer_const_mask = base->pointer_const_mask; - type->is_const_qualified = typedef_const || base->is_const_qualified; - type->is_volatile_qualified = - typedef_volatile || base->is_volatile_qualified; - - /* `const ptr_t` qualifies the pointer that the base typedef hides, not its - * pointee. - */ - if (typedef_const && base->ptr_level && base->ptr_level <= 32 && - !base->func_signature && !base->pointee_func_signature) { - type->is_const_qualified = base->is_const_qualified; - type->pointer_const_mask |= 1U << (base->ptr_level - 1); - } - type->is_unsigned = base->is_unsigned; - type->is_floating = base->is_floating; - type->is_signed_char = base->is_signed_char; - type->is_bool = base->is_bool; - type->array_size = base->array_size; - type->array_dim2 = base->array_dim2; - type->array_dim3 = base->array_dim3; - type->array_dim4 = base->array_dim4; - type->array_element_ptr_level = base->array_element_ptr_level; - type->array_element_type = base->array_element_type; - type->func_signature = base->func_signature; - - /* A tag is not a typedef name. As for `typedef struct S alias`, the alias - * reaches the record through base_struct, which also sees a later - * completion of the tag. - */ - if (base->base_type == TYPE_struct || base->base_type == TYPE_union) { - type->base_type = TYPE_typedef; - type->base_struct = (type_t *) base; - type->is_union = base->base_type == TYPE_union; - } - - /* Handle pointer types in typedef: typedef char *string; */ - unsigned int star_const_mask = 0; - - while (lex_accept(T_asterisk)) { - type->ptr_level++; - type->size = PTR_SIZE; - while (true) { - if (lex_accept(T_const)) { - if (type->ptr_level <= 32) { - type->pointer_const_mask |= 1U << (type->ptr_level - 1); - star_const_mask |= 1U << (type->ptr_level - 1); - } - } else if (lex_accept(T_volatile)) { - /* As for an object declarator, a volatile pointer marks the - * whole declaration volatile. - */ - type->is_volatile_qualified = true; - } else if (lex_accept(T_restrict)) { - ; - } else - break; - } - } - - if (type->ptr_level == 1) - alias_callback_array_pointer(type, base, star_const_mask); - - /* A parenthesized declarator is the function-pointer form: `typedef int - * (*callback_t)(int)`. Parse it through the normal declarator reader so its - * prototype has exactly the same shape as an object declaration, then - * retain that syntax-only signature on the alias for each later object or - * parameter declaration. - */ - if (lex_peek(T_open_bracket, NULL)) { - var_t declarator = {0}; - bool saved_sizeof_signature = parsing_sizeof_function_signature; - - declarator.type = (type_t *) base; - declarator.scope = block; - declarator.ptr_level = type->ptr_level; - - /* A callback typedef carries only a function signature. Its floating - * parameters do not materialize values until a call, which remains - * rejected by the ordinary floating gates. - */ - parsing_sizeof_function_signature = true; - read_inner_var_decl(&declarator, false, false, false); - parsing_sizeof_function_signature = saved_sizeof_signature; - - /* Without a parameter list or an array suffix the parentheses only - * group pointers: `typedef int (*int_ptr)` is `typedef int *int_ptr`. - */ - if (!declarator.is_func && !declarator.array_size && - !declarator.has_unsized_array && !declarator.pointee_array_size && - !base->func_signature && !base->array_size) { - strncpy(type->type_name, declarator.var_name, MAX_TYPE_LEN - 1); - type->type_name[MAX_TYPE_LEN - 1] = '\0'; - type->ptr_level = declarator.ptr_level; - type->size = PTR_SIZE; - type->pointer_const_mask |= declarator.pointer_const_mask; - if (declarator.is_volatile) - type->is_volatile_qualified = true; - - /* `typedef int (**slot_t)(int)` names a callback slot. */ - type->pointee_func_signature = declarator.pointee_func_signature; - return; - } - - /* `typedef int (*row_ptr)[2]` points to a whole row. As a block-scope - * alias does, keep the pointer-sized descriptor and carry the row - * bounds and element separately. The spelled `int (*(*rows_t)[2])(int)` - * reads its row against the unnamed callback type, which the declarator - * then carries. - */ - if (declarator.type && declarator.type->func_signature && - !declarator.type->is_direct_function_type) - base = declarator.type; - if (!declarator.is_func && - declarator.has_direct_pointee_array_declarator && - !declarator.array_size && !base->array_size && - (!base->func_signature || - (!base->is_direct_function_type && !base->ptr_level)) && - !base->pointee_func_signature && - declarator.ptr_level == - declarator.pointee_array_element_ptr_level + 1 && - declarator.pointee_array_element_ptr_level <= 1) { - strncpy(type->type_name, declarator.var_name, MAX_TYPE_LEN - 1); - type->type_name[MAX_TYPE_LEN - 1] = '\0'; - - /* A callback typedef base, `fn_t (*rows_t)[2]`, is the row's - * element with its prototype and qualifiers; the row pointer itself - * is no callback. - */ - if (base->func_signature) - alias_callback_row_pointer(type, base, 0); - - /* A pointer typedef base, `ptr_t (*rows_t)[2]`, is spelled out as - * `int *(*rows_t)[2]`: the declarator already counts its stars on - * the row's element, which then points to the base's pointee. - */ - type->ptr_level = declarator.ptr_level; - type->size = PTR_SIZE; - type->pointer_const_mask |= declarator.pointer_const_mask; - type->pointee_array_size = declarator.pointee_array_size; - type->pointee_array_dim2 = declarator.pointee_array_dim2; - type->pointee_array_dim3 = declarator.pointee_array_dim3; - type->pointee_array_dim4 = declarator.pointee_array_dim4; - type->pointee_array_element_ptr_level = - declarator.pointee_array_element_ptr_level; - type->pointee_array_element_type = - base->ptr_level - ? pointee_type_from_pointer_typedef((type_t *) base) - : (type_t *) base; - return; - } - if (!declarator.is_func) - error_at( - "Typedef parenthesized declarator must be a function " - "pointer", - cur_token_loc()); - - /* `typedef int (*get_t(void))(int)` names a function type whose return - * is a callback, which the declarator reader already built. - */ - if (declarator.is_direct_function_declarator) { - memcpy(type, declarator.type, sizeof(type_t)); - strncpy(type->type_name, declarator.var_name, MAX_TYPE_LEN - 1); - type->type_name[MAX_TYPE_LEN - 1] = '\0'; - return; - } - strncpy(type->type_name, declarator.var_name, MAX_TYPE_LEN - 1); - type->type_name[MAX_TYPE_LEN - 1] = '\0'; - type->size = PTR_SIZE; - type->alignment = PTR_SIZE; - type->func_signature = declarator.func_signature; - - /* `typedef int (*row[2])(int)` is an array typedef whose elements are - * callback pointers. The signature describes each element, while the - * bounds are needed later when a pointer-to-row is indexed (including - * after a call result). - */ - type->array_size = declarator.array_size; - type->array_dim2 = declarator.array_dim2; - type->array_dim3 = declarator.array_dim3; - type->array_dim4 = declarator.array_dim4; - type->array_element_ptr_level = declarator.array_size ? 1 : 0; - - /* Stars before the parenthesized callback declarator belong to the - * callback's return type. That depth is retained in its parsed - * signature; the typedef alias itself is the pointer-sized callback - * object, not a derived pointer alias. - */ - type->ptr_level = 0; - type->pointer_const_mask = declarator.pointer_const_mask; - type->is_volatile_qualified = declarator.is_volatile; - return; - } - - lex_ident_n(T_identifier, type->type_name, MAX_TYPE_LEN); - - /* `typedef int unary_t(int)` names a function type. Mirror the block-scope - * direct function alias: its scalar or void base is only the return type, - * and the prototype stays on the descriptor. - */ - if (lex_peek(T_open_bracket, NULL) && !base->ptr_level && - !base->array_size && !base->func_signature && !is_record_type(base) && - !base->is_floating) { - func_t *func = arena_alloc_func(); - - /* The stars of `char *name_t(void)` belong to the return type. */ - func->return_def.type = (type_t *) base; - func->return_def.ptr_level = type->ptr_level; - func->return_def.pointer_const_mask = type->pointer_const_mask; - func->return_def.scope = block; - type->ptr_level = 0; - type->pointer_const_mask = 0; - type->size = base->size; - read_parameter_list_decl(func, true); - type->base_type = TYPE_typedef; - type->func_signature = func; - type->is_direct_function_type = true; - return; + var_t decl = {0}; + init_type_name_decl(&decl, (type_t *) base, typedef_const, + typedef_volatile); + decl.scope = block; + decl.is_const_qualified |= base->is_const_qualified; + decl.pointer_volatile_mask |= base->pointer_volatile_mask; + decl.pointee_func_signature = base->pointee_func_signature; + decl.is_const_pointer |= typedef_const && base->pointee_func_signature; + bool saved = parsing_sizeof_function_signature; + parsing_sizeof_function_signature = true; + bool saved_typedef = parsing_block_typedef_declarator; + token_t *head = cur_token->next; + while (head->kind == T_asterisk || head->kind == T_const || + head->kind == T_volatile || head->kind == T_restrict) + head = head->next; + parsing_block_typedef_declarator = + head->kind != T_open_bracket && + (base->array_size || (!base->ptr_level && !base->func_signature && + !is_record_type(base) && !base->is_floating)); + if (base->base_type == TYPE_void && !base->ptr_level && + !base->func_signature && !base->pointee_func_signature && + lex_peek(T_identifier, NULL) && + (head->next->kind == T_comma || head->next->kind == T_semicolon)) { + char name[MAX_ID_LEN]; + lex_ident(T_identifier, name); + decl.var_name = intern_string(name); + } else + read_inner_var_decl(&decl, false, false, false); + parsing_block_typedef_declarator = saved_typedef; + parsing_sizeof_function_signature = saved; + if (decl.is_direct_function_declarator && + !decl.parenthesized_function_pointer_level && + function_signature_has_floating(decl.func_signature)) + error_at("Floating point types are not yet supported", cur_token_loc()); + if (decl.is_direct_function_declarator && base->array_size) + error_at("Unexpected token", cur_token_loc()); + if (decl.has_direct_pointee_array_declarator && decl.type->func_signature && + !decl.type->is_direct_function_type) + base = decl.type; + if (!decl.is_func && + (head->kind == T_open_bracket || + decl.parenthesized_function_pointer_level || + decl.has_direct_pointee_array_declarator) && + !((!decl.array_size && !decl.has_unsized_array && + !decl.pointee_array_size && !base->func_signature && + !base->array_size) || + (decl.has_direct_pointee_array_declarator && !decl.array_size && + !base->array_size && + (!base->func_signature || + (!base->is_direct_function_type && !base->ptr_level)) && + !base->pointee_func_signature && + effective_pointer_depth(&decl) == + decl.pointee_array_element_ptr_level + base->ptr_level + 1 && + decl.pointee_array_element_ptr_level + base->ptr_level <= 1))) + error_at("Typedef parenthesized declarator must be a function pointer", + cur_token_loc()); + if (base->is_direct_function_type && decl.array_size && decl.is_func && + !decl.parenthesized_function_pointer_level) { + decl.type = (type_t *) base; + decl.ptr_level = 1; + decl.is_func = false; } - - /* A typedef declarator may wrap an existing array typedef: `typedef row - * matrix[2]`. Gather its leading bounds first, then prepend them to the - * base alias's bounds instead of overwriting the inner extent. + if (decl.has_unsized_array) + error_at("Typedef array needs a positive bound", cur_token_loc()); + type_t *alias = typedef_alias_from_declarator(&decl, typedef_const, + typedef_volatile, true); + strncpy(alias->type_name, decl.var_name, MAX_TYPE_LEN - 1); + alias->type_name[MAX_TYPE_LEN - 1] = '\0'; + + /* A repeated typedef must name a compatible type; the first binding is the + * one later uses resolve to, so a conflicting one would be ignored. */ - fixed_array_shape_t base_shape = fixed_array_shape_from_type(type); - fixed_array_shape_t decl_shape = {0}; - while (lex_accept(T_open_square)) { - int bound; - - if (decl_shape.rank >= MAX_FIXED_ARRAY_RANK) - error_at("Array declarators support at most four dimensions", - cur_token_loc()); - if (lex_peek(T_close_square, NULL)) - error_at("Typedef array needs a positive bound", cur_token_loc()); - bound = read_const_expr(block); - if (bound <= 0) - error_at("Typedef array needs a positive bound", cur_token_loc()); - decl_shape.bounds[decl_shape.rank++] = bound; - lex_expect(T_close_square); - } - if (decl_shape.rank) { - fixed_array_shape_t shape = - fixed_array_shape_prepend(&decl_shape, &base_shape); - - if (!type->ptr_level && !type->size && type->base_struct && - !type->base_struct->size) - error_at("Typedef array element has incomplete record type", - cur_token_loc()); - - fixed_array_shape_to_type(type, &shape); - - /* At the point an array typedef is introduced, ptr_level describes each - * array element. A later alias may add a pointer to the whole array, so - * preserve this separately. - */ - type->array_element_ptr_level = type->ptr_level; - type->array_element_type = - base->array_element_type - ? base->array_element_type - : (base->ptr_level - ? pointee_type_from_pointer_typedef((type_t *) base) - : (type_t *) base); - } + void *previous = NULL; + if (find_block_ordinary(CURRENT_TU_SCOPE, alias->type_name, + ORDINARY_TYPEDEF, &previous) == ORDINARY_TYPEDEF && + !compatible_decl_type(((typedef_binding_t *) previous)->type, alias)) + error_at("conflicting types for typedef", cur_token_loc()); + add_tu_typedef(alias); } /* The declarator list of a file-scope typedef whose specifier resolved to @@ -1311,7 +964,7 @@ static void read_global_typedef(block_t *block) if (lex_peek(T_open_curly, NULL)) record = read_record_body(NULL, kind, has_tag, token); else if (has_tag) - tag = local_record_tag(token, GLOBAL_BLOCK, kind); + tag = local_record_tag(token, CURRENT_TU_SCOPE, kind); read_type_qualifiers(&typedef_const, &typedef_volatile, false); bool is_plain = global_record_typedef_declarator_is_plain(); @@ -1321,8 +974,8 @@ static void read_global_typedef(block_t *block) lex_ident_n(T_identifier, type->type_name, MAX_TYPE_LEN); /* A defining alias carries the layout itself; a forward one reaches the - * tag through base_struct, which in 'find_type' also sees a later - * completion of it. + * tag through base_struct, which in resolve_type_alias() also sees a + * later completion of it. */ int size = record ? record->size : 0; int alignment = record ? record->alignment : 1; @@ -1350,6 +1003,7 @@ static void read_global_typedef(block_t *block) /* Only the alias is qualified; the tag must stay plain. */ type->is_const_qualified = typedef_const; type->is_volatile_qualified = typedef_volatile; + add_tu_typedef(type); if (!is_plain) { read_global_typedef_declarators(block, typedef_const, @@ -1387,7 +1041,7 @@ static void read_global_typedef(block_t *block) void read_global_statement(void) { char token[MAX_ID_LEN]; - block_t *block = GLOBAL_BLOCK; /* global block */ + block_t *block = GLOBAL_BLOCK; /* merged object/function namespace */ bool is_const = false; bool is_static = false; bool is_extern = false; @@ -1461,7 +1115,7 @@ void read_global_statement(void) /* variable declaration using existing record tag? */ if (!lex_peek(T_open_curly, NULL)) { - type = local_record_tag(token, GLOBAL_BLOCK, kind); + type = local_record_tag(token, CURRENT_TU_SCOPE, kind); /* A declaration with no declarator only declares the tag. At file * scope a repeated one names the same type, so it is valid whether @@ -1471,7 +1125,7 @@ void read_global_statement(void) return; read_global_declarator_list(block, type, is_const, is_static, - is_volatile, is_extern, true); + is_volatile, is_extern); return; } @@ -1482,7 +1136,7 @@ void read_global_statement(void) */ if (!lex_accept(T_semicolon)) read_global_declarator_list(block, type, is_const, is_static, - is_volatile, is_extern, true); + is_volatile, is_extern); } else if (lex_peek(T_enum, NULL)) { bool is_definition; type_t *type = read_enum_specifier(NULL, &is_definition); @@ -1497,7 +1151,7 @@ void read_global_statement(void) next_token_loc()); if (!is_definition || !lex_accept(T_semicolon)) read_global_declarator_list(block, type, is_const, is_static, - is_volatile, is_extern, false); + is_volatile, is_extern); } else if (lex_accept(T_typedef)) { read_global_typedef(block); } else if (lex_peek(T_identifier, NULL) || lex_peek(T_signed, NULL) || @@ -1508,6 +1162,21 @@ void read_global_statement(void) error_at("Syntax error in global statement", next_token_loc()); } +static type_t *add_builtin_scalar(const char *name, + base_type_t base_type, + int size, + bool is_unsigned, + bool is_signed_char) +{ + type_t *type = add_named_type((char *) name); + + type->base_type = base_type; + type->size = size; + type->is_unsigned = is_unsigned; + type->is_signed_char = is_signed_char; + return type; +} + void parse_internal(void) { /* set starting point of global stack manually */ @@ -1517,7 +1186,7 @@ void parse_internal(void) * occupy a full target pointer, not the historic 32-bit word. */ GLOBAL_FUNC->stack_size = PTR_SIZE; - GLOBAL_FUNC->bbs = arena_calloc(BB_ARENA, 1, sizeof(basic_block_t)); + GLOBAL_FUNC->bbs = arena_alloc_bb(); GLOBAL_FUNC->bbs->belong_to = GLOBAL_FUNC; /* Prevent nullptr deref in RA */ GLOBAL_FUNC->bbs->elf_offset = -1; /* not yet emitted */ @@ -1593,9 +1262,7 @@ void parse_internal(void) TY_ushort->is_unsigned = true; /* Unlike `long`, which deliberately shares the current 32-bit int ABI, long - * long has a distinct type and an eight-byte object representation. Parser - * admission and target lowering are staged separately so 32-bit backends - * never silently truncate it. + * long has a distinct type and an eight-byte object representation. */ TY_long_long = add_named_type("long long"); TY_long_long->base_type = TYPE_long_long; @@ -1610,14 +1277,10 @@ void parse_internal(void) * builtin type table so declarations, casts, sizeof, and prototypes use the * same pointer-width representation as the rest of the compiler. */ - type_t *TY_size = add_named_type("size_t"); - TY_size->base_type = PTR_SIZE == 8 ? TYPE_long_long : TYPE_long; - TY_size->size = PTR_SIZE; - TY_size->is_unsigned = true; - - type_t *TY_ptrdiff = add_named_type("ptrdiff_t"); - TY_ptrdiff->base_type = PTR_SIZE == 8 ? TYPE_long_long : TYPE_long; - TY_ptrdiff->size = PTR_SIZE; + add_builtin_scalar("size_t", PTR_SIZE == 8 ? TYPE_long_long : TYPE_long, + PTR_SIZE, true, false); + add_builtin_scalar("ptrdiff_t", PTR_SIZE == 8 ? TYPE_long_long : TYPE_long, + PTR_SIZE, false, false); /* belongs to C99's freestanding library. Its va_list is the * compiler ABI's int-based cursor, so retain that scalar base while @@ -1631,124 +1294,47 @@ void parse_internal(void) /* The execution wide-character type is int until wide literal lowering is * implemented; declaring the C99 typedef remains useful independently. */ - type_t *TY_wchar = add_named_type("wchar_t"); - TY_wchar->base_type = TYPE_int; - TY_wchar->size = TY_int->size; - - type_t *TY_sig_atomic = add_named_type("sig_atomic_t"); - TY_sig_atomic->base_type = TYPE_int; - TY_sig_atomic->size = TY_int->size; - - type_t *TY_wint = add_named_type("wint_t"); - TY_wint->base_type = TYPE_int; - TY_wint->size = TY_int->size; - TY_wint->is_unsigned = true; + add_builtin_scalar("wchar_t", TYPE_int, TY_int->size, false, false); + add_builtin_scalar("sig_atomic_t", TYPE_int, TY_int->size, false, false); + add_builtin_scalar("wint_t", TYPE_int, TY_int->size, true, false); /* C99 aliases share the target scalar representations. Keep them * named in the builtin table so declarations, casts, sizeof, and prototypes * use the same ABI metadata as their underlying types. */ - type_t *TY_int8 = add_named_type("int8_t"); - TY_int8->base_type = TYPE_char; - TY_int8->size = 1; - TY_int8->is_signed_char = true; - type_t *TY_uint8 = add_named_type("uint8_t"); - TY_uint8->base_type = TYPE_char; - TY_uint8->size = 1; - TY_uint8->is_unsigned = true; - type_t *TY_int16 = add_named_type("int16_t"); - TY_int16->base_type = TYPE_short; - TY_int16->size = 2; - type_t *TY_uint16 = add_named_type("uint16_t"); - TY_uint16->base_type = TYPE_short; - TY_uint16->size = 2; - TY_uint16->is_unsigned = true; - type_t *TY_int32 = add_named_type("int32_t"); - TY_int32->base_type = TYPE_int; - TY_int32->size = 4; - type_t *TY_uint32 = add_named_type("uint32_t"); - TY_uint32->base_type = TYPE_int; - TY_uint32->size = 4; - TY_uint32->is_unsigned = true; - type_t *TY_int64 = add_named_type("int64_t"); - TY_int64->base_type = TYPE_long_long; - TY_int64->size = 8; - type_t *TY_uint64 = add_named_type("uint64_t"); - TY_uint64->base_type = TYPE_long_long; - TY_uint64->size = 8; - TY_uint64->is_unsigned = true; - type_t *TY_intmax = add_named_type("intmax_t"); - TY_intmax->base_type = TYPE_long_long; - TY_intmax->size = 8; - type_t *TY_uintmax = add_named_type("uintmax_t"); - TY_uintmax->base_type = TYPE_long_long; - TY_uintmax->size = 8; - TY_uintmax->is_unsigned = true; - type_t *TY_intptr = add_named_type("intptr_t"); - TY_intptr->base_type = PTR_SIZE == 8 ? TYPE_long_long : TYPE_long; - TY_intptr->size = PTR_SIZE; - type_t *TY_uintptr = add_named_type("uintptr_t"); - TY_uintptr->base_type = PTR_SIZE == 8 ? TYPE_long_long : TYPE_long; - TY_uintptr->size = PTR_SIZE; - TY_uintptr->is_unsigned = true; - - type_t *TY_int_least8 = add_named_type("int_least8_t"); - TY_int_least8->base_type = TY_int8->base_type; - TY_int_least8->size = TY_int8->size; - TY_int_least8->is_signed_char = true; - type_t *TY_uint_least8 = add_named_type("uint_least8_t"); - TY_uint_least8->base_type = TY_uint8->base_type; - TY_uint_least8->size = TY_uint8->size; - TY_uint_least8->is_unsigned = true; - type_t *TY_int_least16 = add_named_type("int_least16_t"); - TY_int_least16->base_type = TYPE_short; - TY_int_least16->size = 2; - type_t *TY_uint_least16 = add_named_type("uint_least16_t"); - TY_uint_least16->base_type = TYPE_short; - TY_uint_least16->size = 2; - TY_uint_least16->is_unsigned = true; - type_t *TY_int_least32 = add_named_type("int_least32_t"); - TY_int_least32->base_type = TYPE_int; - TY_int_least32->size = 4; - type_t *TY_uint_least32 = add_named_type("uint_least32_t"); - TY_uint_least32->base_type = TYPE_int; - TY_uint_least32->size = 4; - TY_uint_least32->is_unsigned = true; - type_t *TY_int_least64 = add_named_type("int_least64_t"); - TY_int_least64->base_type = TYPE_long_long; - TY_int_least64->size = 8; - type_t *TY_uint_least64 = add_named_type("uint_least64_t"); - TY_uint_least64->base_type = TYPE_long_long; - TY_uint_least64->size = 8; - TY_uint_least64->is_unsigned = true; - type_t *TY_int_fast8 = add_named_type("int_fast8_t"); - TY_int_fast8->base_type = TYPE_int; - TY_int_fast8->size = 4; - type_t *TY_uint_fast8 = add_named_type("uint_fast8_t"); - TY_uint_fast8->base_type = TYPE_int; - TY_uint_fast8->size = 4; - TY_uint_fast8->is_unsigned = true; - type_t *TY_int_fast16 = add_named_type("int_fast16_t"); - TY_int_fast16->base_type = TYPE_int; - TY_int_fast16->size = 4; - type_t *TY_uint_fast16 = add_named_type("uint_fast16_t"); - TY_uint_fast16->base_type = TYPE_int; - TY_uint_fast16->size = 4; - TY_uint_fast16->is_unsigned = true; - type_t *TY_int_fast32 = add_named_type("int_fast32_t"); - TY_int_fast32->base_type = TYPE_int; - TY_int_fast32->size = 4; - type_t *TY_uint_fast32 = add_named_type("uint_fast32_t"); - TY_uint_fast32->base_type = TYPE_int; - TY_uint_fast32->size = 4; - TY_uint_fast32->is_unsigned = true; - type_t *TY_int_fast64 = add_named_type("int_fast64_t"); - TY_int_fast64->base_type = TYPE_long_long; - TY_int_fast64->size = 8; - type_t *TY_uint_fast64 = add_named_type("uint_fast64_t"); - TY_uint_fast64->base_type = TYPE_long_long; - TY_uint_fast64->size = 8; - TY_uint_fast64->is_unsigned = true; + type_t *TY_int8 = add_builtin_scalar("int8_t", TYPE_char, 1, false, true); + type_t *TY_uint8 = add_builtin_scalar("uint8_t", TYPE_char, 1, true, false); + add_builtin_scalar("int16_t", TYPE_short, 2, false, false); + add_builtin_scalar("uint16_t", TYPE_short, 2, true, false); + add_builtin_scalar("int32_t", TYPE_int, 4, false, false); + add_builtin_scalar("uint32_t", TYPE_int, 4, true, false); + add_builtin_scalar("int64_t", TYPE_long_long, 8, false, false); + add_builtin_scalar("uint64_t", TYPE_long_long, 8, true, false); + add_builtin_scalar("intmax_t", TYPE_long_long, 8, false, false); + add_builtin_scalar("uintmax_t", TYPE_long_long, 8, true, false); + add_builtin_scalar("intptr_t", PTR_SIZE == 8 ? TYPE_long_long : TYPE_long, + PTR_SIZE, false, false); + add_builtin_scalar("uintptr_t", PTR_SIZE == 8 ? TYPE_long_long : TYPE_long, + PTR_SIZE, true, false); + + add_builtin_scalar("int_least8_t", TY_int8->base_type, TY_int8->size, false, + true); + add_builtin_scalar("uint_least8_t", TY_uint8->base_type, TY_uint8->size, + true, false); + add_builtin_scalar("int_least16_t", TYPE_short, 2, false, false); + add_builtin_scalar("uint_least16_t", TYPE_short, 2, true, false); + add_builtin_scalar("int_least32_t", TYPE_int, 4, false, false); + add_builtin_scalar("uint_least32_t", TYPE_int, 4, true, false); + add_builtin_scalar("int_least64_t", TYPE_long_long, 8, false, false); + add_builtin_scalar("uint_least64_t", TYPE_long_long, 8, true, false); + add_builtin_scalar("int_fast8_t", TYPE_int, 4, false, false); + add_builtin_scalar("uint_fast8_t", TYPE_int, 4, true, false); + add_builtin_scalar("int_fast16_t", TYPE_int, 4, false, false); + add_builtin_scalar("uint_fast16_t", TYPE_int, 4, true, false); + add_builtin_scalar("int_fast32_t", TYPE_int, 4, false, false); + add_builtin_scalar("uint_fast32_t", TYPE_int, 4, true, false); + add_builtin_scalar("int_fast64_t", TYPE_long_long, 8, false, false); + add_builtin_scalar("uint_fast64_t", TYPE_long_long, 8, true, false); /* builtin type _Bool was introduced in C99 specification, it is more * well-known as macro type bool, which is defined in (in @@ -1759,35 +1345,27 @@ void parse_internal(void) TY_bool->size = 1; TY_bool->is_bool = true; + builtin_types_idx = types_idx; + GLOBAL_BLOCK = add_block(NULL, NULL); /* global block */ - elf_add_symbol("", 0); /* undef symbol */ + CURRENT_TU_SCOPE = add_block(NULL, NULL); + reset_tu_ordinary_index(); + elf_add_symbol("", 0); /* undef symbol */ if (dynlink) { - /* In dynamic mode, __syscall won't be implemented. - * - * Simply declare a 'syscall' function as follows if the program needs - * to use 'syscall': - * - * int syscall(int number, ...); - * - * shecc will treat it as an external function, and the compiled program - * will eventually use the implementation provided by the external C - * library. - * - * If shecc supports the 'long' data type in the future, it would be - * better to declare syscall using its original prototype: - * - * long syscall(long number, ...); + /* Dynamic callers declare the libc entry with a machine-word ABI: + * intptr_t syscall(intptr_t number, ...); intptr_t preserves + * pointer-valued results on every target. */ } else { /* Linux syscall */ func_t *func = add_func("__syscall", true); - func->return_def.type = TY_int; + func->return_def.type = PTR_SIZE == 8 ? TY_long_long : TY_int; func->num_params = 0; func->va_args = 1; func->bbs = NULL; /* Otherwise, allocate a basic block to implement in static mode. */ - func->bbs = arena_calloc(BB_ARENA, 1, sizeof(basic_block_t)); + func->bbs = arena_alloc_bb(); func->bbs->elf_offset = -1; /* not yet emitted */ } @@ -1801,7 +1379,13 @@ void parse_internal(void) /* lexer initialization */ do { + if (lex_accept(T_translation_unit)) { + CURRENT_TU_SCOPE = add_block(NULL, NULL); + reset_tu_ordinary_index(); + continue; + } read_global_statement(); + release_parsed_tokens(); } while (!lex_accept(T_eof)); /* Aggregate returns use shecc's internal destination-pointer convention, diff --git a/src/parser-init.c b/src/parser-init.c index a7333cef..6e8cb772 100644 --- a/src/parser-init.c +++ b/src/parser-init.c @@ -13,6 +13,7 @@ */ void read_parameter_list_decl(func_t *func, bool anon); +void read_wstring_param(block_t *parent, basic_block_t *bb); void read_indirect_call(var_t *callee, block_t *parent, basic_block_t **bb); var_t *integer_promote_operand(block_t *parent, basic_block_t **bb, var_t *var); var_t *resolve_global_declarator(block_t *block, @@ -28,18 +29,28 @@ int read_const_expr(block_t *scope); var_t *read_wide_global_literal_expression(block_t *parent, basic_block_t *bb, block_t *scope); +var_t *read_wide_global_literal_tail(block_t *parent, + basic_block_t *bb, + block_t *scope, + var_t *first); var_t *read_wide_global_literal_primary(block_t *parent, basic_block_t *bb, block_t *scope); bool subscripted_string_literal_starts_here(void); -bool string_element_appears_before_initializer_end(token_t *token); var_t *read_string_literal_element_address(block_t *parent, basic_block_t *bb, block_t *scope); -bool string_address_offset_starts_here(void); -var_t *read_string_address_offset(block_t *parent, - basic_block_t *bb, - block_t *scope); + +static void parse_array_init_at(var_t *var, + block_t *parent, + basic_block_t **bb, + var_t *base_addr, + bool sparse_zero); + +static bool initializer_separator(bool comma_consumed) +{ + return comma_consumed || lex_accept(T_comma); +} /* Forward declaration for ternary handling used by initializers */ void read_ternary_operation(block_t *parent, basic_block_t **bb); @@ -47,43 +58,37 @@ void read_ternary_operation(block_t *parent, basic_block_t **bb); /* Parse array initializer to determine size for implicit arrays and optionally * emit initialization code. */ -var_t *compute_element_address(block_t *parent, - basic_block_t **bb, - var_t *base_addr, - int index, - int elem_size) +static var_t *compute_byte_offset_address(block_t *parent, + basic_block_t **bb, + var_t *base_addr, + int byte_offset) { - if (index == 0) + if (byte_offset == 0) return base_addr; - var_t *offset = require_var(parent); - offset->var_name = gen_name(); - offset->init_val = index * elem_size; - add_insn(parent, *bb, OP_load_constant, offset, NULL, NULL, 0, NULL); + var_t *offset = load_constant(parent, *bb, byte_offset, TY_int); - var_t *addr = require_var(parent); - addr->var_name = gen_name(); + var_t *addr = require_named_var(parent); add_insn(parent, *bb, OP_add, addr, base_addr, offset, 0, NULL); return addr; } +var_t *compute_element_address(block_t *parent, + basic_block_t **bb, + var_t *base_addr, + int index, + int elem_size) +{ + return compute_byte_offset_address(parent, bb, base_addr, + index * elem_size); +} + var_t *compute_field_address(block_t *parent, basic_block_t **bb, var_t *struct_addr, const var_t *field) { - if (field->offset == 0) - return struct_addr; - - var_t *offset = require_var(parent); - offset->var_name = gen_name(); - offset->init_val = field->offset; - add_insn(parent, *bb, OP_load_constant, offset, NULL, NULL, 0, NULL); - - var_t *addr = require_var(parent); - addr->var_name = gen_name(); - add_insn(parent, *bb, OP_add, addr, struct_addr, offset, 0, NULL); - return addr; + return compute_byte_offset_address(parent, bb, struct_addr, field->offset); } /* A record assignment is a value copy, not the scalar OP_assign used for @@ -100,6 +105,27 @@ bool is_record_object(const var_t *var) var->type->num_fields > 0)); } +/* Choose the widest indirect access allowed by the remaining bytes and + * alignment. Widths descend by halves, matching backend scalar stores. + */ +static int byte_access_width(int remaining, int alignment, int offset) +{ + for (int width = 4; width > 1; width /= 2) + if (remaining >= width && alignment >= width && !(alignment % width) && + !(offset % width)) + return width; + return 1; +} + +static int initializer_alignment(const var_t *var) +{ + if (!var) + return 1; + return var->is_func || effective_pointer_depth(var) + ? PTR_SIZE + : alignment_type(var->type); +} + /* Copy size bytes between two known record addresses in 4-, 2- and 1-byte * slices, the widest indirect accesses every backend encodes. A record is an * object, not an integer value, so one OP_write of its whole size would leave @@ -112,18 +138,12 @@ void emit_record_copy_between(block_t *parent, int size) { for (int offset = 0; offset < size;) { - int width = 1; - if (size - offset >= 4) - width = 4; - else if (size - offset >= 2) - width = 2; + int width = byte_access_width(size - offset, PTR_SIZE, offset); var_t *src_part = compute_element_address(parent, bb, src_addr, offset, 1); var_t *dest_part = compute_element_address(parent, bb, dest_addr, offset, 1); - var_t *value = require_var(parent); - - value->var_name = gen_name(); + var_t *value = require_named_var(parent); add_insn(parent, *bb, OP_read, value, src_part, NULL, width, NULL); add_insn(parent, *bb, OP_write, NULL, dest_part, value, width, NULL); offset += width; @@ -140,8 +160,7 @@ var_t *record_value_address(block_t *parent, basic_block_t *bb, var_t *record) if (record->defers_record_copy) return record->compound_literal_address; - address = require_ref_var(parent, record->type, 0); - address->var_name = gen_name(); + address = name_var(require_ref_var(parent, record->type, 0)); add_insn(parent, bb, OP_address_of, address, record, NULL, 0, NULL); return address; } @@ -188,7 +207,6 @@ void emit_object_assignment(block_t *parent, * as `int (*fn)(int) = target;`. */ var_t *dest_addr = require_ref_var(parent, dest->type, dest->ptr_level); - dest_addr->var_name = gen_name(); add_insn(parent, *bb, OP_address_of, dest_addr, dest, NULL, 0, NULL); add_insn(parent, *bb, OP_write, NULL, dest_addr, src, PTR_SIZE, NULL); } else { @@ -198,10 +216,8 @@ void emit_object_assignment(block_t *parent, } type_t *read_type_name_specifiers(block_t *scope); +void read_type_name_decl(block_t *scope, var_t *decl); int read_const_expr_operand(block_t *scope); -int read_global_address_offset(block_t *scope, - block_t *parent, - basic_block_t *bb); /* While the operand of a pointer cast in a static initializer is read, the size * of what the cast pointer points to, and 0 otherwise. An offset that follows @@ -210,15 +226,8 @@ int read_global_address_offset(block_t *scope, */ int global_pointer_cast_stride = 0; -/* While the operand of a cast to a function pointer type in a static - * initializer is read, the prototype that cast names, and NULL otherwise. The - * cast, not the designated function, is converted to the initialized object. - */ -func_t *global_function_cast_signature = NULL; - bool abstract_function_pointer_follows(void); -func_t *read_abstract_function_pointer(type_t *type, - int ptr_level, +func_t *read_abstract_function_pointer(const var_t *return_decl, int *pointer_level); /* If a cast to a function pointer type, `(int (*)(int))` or `(callback_t)`, @@ -232,22 +241,17 @@ func_t *read_global_function_pointer_cast(block_t *scope) type_t *type; int stars = 0; int pointer_level = 0; + var_t return_decl; if (!lex_accept(T_open_bracket)) return NULL; - type = read_type_name_specifiers(scope); + read_type_name_decl(scope, &return_decl); + type = return_decl.type; + stars = return_decl.ptr_level; if (type) { - while (lex_accept(T_const) || lex_accept(T_volatile)) - ; - while (lex_accept(T_asterisk)) { - stars++; - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; - } if (abstract_function_pointer_follows()) { signature = - read_abstract_function_pointer(type, stars, &pointer_level); + read_abstract_function_pointer(&return_decl, &pointer_level); if (pointer_level != 1) signature = NULL; } else if (type->func_signature && !type->is_direct_function_type && @@ -264,8 +268,6 @@ func_t *read_global_function_pointer_cast(block_t *scope) return signature; } -static token_t *matching_close_bracket(token_t *open); - /* Say whether cur_token->next opens a cast to an object pointer type in a * static initializer. C99 6.6 lets an address constant and an integer constant * be converted by such a cast. @@ -292,18 +294,14 @@ bool global_pointer_cast_starts_here(block_t *scope) /* A pointer to function pointers, `(int (**)(void))`, points to * pointer objects: two or more stars, then a parameter list. */ - int stars = 0; - - for (token = token->next; - token && - (token->kind == T_asterisk || token->kind == T_const || - token->kind == T_volatile || token->kind == T_restrict); - token = token->next) - stars += token->kind == T_asterisk; + int stars; + + token = function_pointer_declarator_suffix(token, &stars); if (stars < 2 || !token || token->kind != T_close_bracket || !(token = token->next) || token->kind != T_open_bracket) return false; - token = matching_close_bracket(token); + token = + skip_balanced_delimiter(token, T_open_bracket, T_close_bracket); if (!token || !(token = token->next)) return false; return token->kind == T_close_bracket; @@ -340,53 +338,50 @@ bool global_pointer_cast_starts_here(block_t *scope) return token && has_type && is_pointer && (!function_type || depth > 1); } -/* Consume a cast that global_pointer_cast_starts_here() recognized and return - * the size of what its pointer type points to. @slot_signature receives the - * prototype of the function pointers that type points to, as `(int (**)(void))` - * or `(callback_t *)` do, and NULL for any other pointer. +/* Consume an object-pointer cast, retain its complete destination type, and + * return the pointee stride used by static address arithmetic. */ -int read_global_pointer_cast(block_t *scope, func_t **slot_signature) +int read_global_pointer_cast(block_t *scope, var_t *cast) { - type_t *type; - int depth = 0; - int size; - - *slot_signature = NULL; lex_expect(T_open_bracket); - type = read_type_name_specifiers(scope); - while (lex_accept(T_asterisk)) { - depth++; - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; - } + read_type_name_decl(scope, cast); + type_t *type = cast->type; + int depth = cast->ptr_level; if (!type) error_at("Unknown type in pointer cast", cur_token_loc()); + cast->is_func = false; + cast->func_signature = NULL; - /* A pointer to function pointers, spelled out or through a callback - * typedef, steps over pointer objects. - */ if (abstract_function_pointer_follows()) { int pointer_level; func_t *signature = - read_abstract_function_pointer(type, depth, &pointer_level); - + read_abstract_function_pointer(cast, &pointer_level); lex_expect(T_close_bracket); - if (pointer_level == 2) - *slot_signature = signature; + cast->ptr_level += pointer_level - 1; + cast->pointee_func_signature = signature; return PTR_SIZE; } lex_expect(T_close_bracket); if (type->func_signature && - (type->is_direct_function_type ? depth == 2 - : !type->ptr_level && depth == 1)) - *slot_signature = type->func_signature; + (type->is_direct_function_type ? depth >= 2 + : !type->ptr_level && depth >= 1)) + cast->pointee_func_signature = type->func_signature; + if (cast->pointee_func_signature) { + func_t *signature = cast->pointee_func_signature; + cast->type = signature->return_def.type; + cast->ptr_level = signature->return_def.ptr_level + depth - + type->is_direct_function_type; + return PTR_SIZE; + } if (depth + type->ptr_level > 1 || type->func_signature) return PTR_SIZE; type = pointee_type_from_pointer_typedef(type); - if (type == TY_void) + if (type == TY_void) { + cast->type = TY_void; + cast->ptr_level = 1; return 1; - size = type->size; + } + int size = type->size; if (!size && type->base_struct) size = type->base_struct->size; if (type->array_size) @@ -401,7 +396,9 @@ bool global_address_operand_starts_here(block_t *scope) { char name[MAX_ID_LEN]; - if (lex_peek(T_ampersand, NULL) || lex_peek(T_string, NULL) || + if (lex_peek(T_ampersand, NULL) || + (lex_peek(T_string, NULL) && + !subscripted_string_literal_starts_here()) || global_pointer_cast_starts_here(scope) || grouped_global_function_designator_starts_here(false) || global_function_address_dereference_starts_here()) @@ -415,17 +412,17 @@ bool global_address_operand_starts_here(block_t *scope) return false; } -/* Read an integer constant operand of a pointer cast with @stride and - * @slot_signature, and any offset that follows it, as the address constant the - * cast produces. - */ +static bool wide_global_address_operand(const var_t *value); +bool is_pointer_like_value(var_t *var); + +/* Read a pointer cast's integer or grouped address operand. */ var_t *read_global_cast_integer_address(block_t *parent, basic_block_t *bb, block_t *scope, int stride, func_t *slot_signature) { - var_t *address = require_var(parent); + var_t *address; char name[MAX_ID_LEN]; unsigned int lo; unsigned int hi; @@ -439,20 +436,26 @@ var_t *read_global_cast_integer_address(block_t *parent, lo = read_const_expr_operand(scope); hi = lo & 0x80000000U ? ~0U : 0; } else { + int saved_stride = global_pointer_cast_stride; + global_pointer_cast_stride = 0; var_t *operand = read_wide_global_literal_primary(parent, bb, scope); + global_pointer_cast_stride = saved_stride; + if (wide_global_address_operand(operand)) { + if (operand->is_function_designator) + error_at("incompatible function pointer types", + cur_token_loc()); + operand->is_global_address = true; + operand->address_stride = stride; + return operand; + } lo = operand->init_val; hi = operand->init_val_hi; - if (operand->type->size <= TY_int->size) + if (!is_pointer_like_value(operand) && + operand->type->size <= TY_int->size) hi = operand->type->is_unsigned || !(lo & 0x80000000U) ? 0 : ~0U; } - if (lex_peek(T_plus, NULL) || lex_peek(T_minus, NULL)) { - int offset = read_global_address_offset(scope, parent, bb) * stride; - unsigned int sum = lo + offset; - - hi += (offset < 0 ? ~0U : 0) + (sum < lo); - lo = sum; - } + address = require_var(parent); address->var_name = gen_name(); address->init_val = lo; address->init_val_hi = PTR_SIZE == 8 ? hi : 0; @@ -503,8 +506,9 @@ var_t *read_global_address_designator(block_t *scope, shape = fixed_array_shape_from_var(target); subscripts = 0; } else if (lex_accept(T_open_square)) { - int element_size = - target->ptr_level ? PTR_SIZE : target->type->size; + int element_size = target->ptr_level || target->is_func + ? PTR_SIZE + : target->type->size; int index; if (!target->array_size || subscripts >= shape.rank) @@ -522,33 +526,40 @@ var_t *read_global_address_designator(block_t *scope, object_addr->is_global_address = true; } - /* A trailing offset after a partial multidimensional subscript advances by - * the remaining row or plane, the complete pointed-to object, rather than - * by its scalar leaf. read_global_address_offset() takes the sign as part - * of the constant expression. - */ if (decays && (!target->array_size || subscripts >= shape.rank)) error_at("Global initializer requires a constant address", cur_token_loc()); - int element_size = target->ptr_level ? PTR_SIZE : target->type->size; + int element_size = + target->ptr_level || target->is_func ? PTR_SIZE : target->type->size; int stride = global_pointer_cast_stride ? global_pointer_cast_stride : fixed_array_shape_stride(&shape, subscripts - !decays, element_size); - if (lex_peek(T_plus, NULL) || lex_peek(T_minus, NULL)) { - int index = read_global_address_offset(scope, parent, *bb); - - object_addr = - compute_element_address(parent, bb, object_addr, index, stride); - object_addr->ptr_level = target->ptr_level + 1; - object_addr->is_global_address = true; - } + object_addr->type = target->type; + object_addr->pointee_func_signature = target->pointee_func_signature + ? target->pointee_func_signature + : target->func_signature; object_addr->address_stride = stride; *object = target; return object_addr; } +static var_t *read_global_object_address(block_t *scope, + block_t *parent, + basic_block_t **bb, + var_t *object, + bool decays) +{ + var_t *address = + name_var(require_ref_var(parent, object->type, object->ptr_level)); + + address->is_global_address = true; + add_insn(parent, *bb, OP_address_of, address, object, NULL, 0, NULL); + return read_global_address_designator(scope, parent, bb, &object, address, + decays); +} + /* The block that names in an initializer lowered into @parent resolve in. A * block-scope static is lowered through GLOBAL_BLOCK, yet its designators and * constants belong to the lexical scope of its declaration. @@ -560,36 +571,104 @@ block_t *initializer_name_scope(block_t *parent) return parent; } -/* Nonzero while the constant expression reader has this function read one - * address operand of the expression it is already reading. - */ -int global_tested_operand_depth; +typedef struct { + block_t *scope; + var_t *dest; +} global_constant_context_t; -/* Whether the element ahead is an address constant that an operator only - * tests, as in `"a" && 1`, `1 ? "a" : "b"` or `array || 0`. Such an element is - * a constant expression, not the address itself, so the whole expression - * belongs to the constant expression reader. - */ -static bool global_tested_operand_starts_here(block_t *scope) +static var_t *global_constant_destination; +static int wide_global_unevaluated_depth; + +static void validate_global_compound_address(const var_t *dest, + const var_t *literal); + +static void diagnose_global_function_value(global_constant_context_t *ctx, + var_t *value) { - char name[MAX_ID_LEN]; - token_t *after; + if (!ctx->dest) + return; + if (!ctx->dest->is_func && !ctx->dest->ptr_level && + !(ctx->dest->type && ctx->dest->type->ptr_level)) + error_at("Function address requires a pointer initializer", + cur_token_loc()); + diagnose_function_pointer_conversion(value, ctx->dest); +} - if (global_tested_operand_depth || !cur_token->next) - return false; - if (!lex_peek(T_string, NULL) && - !(lex_peek(T_identifier, name) && - (find_visible_func(name, scope) || find_var(name, scope)))) - return false; - after = cur_token->next->next; - return after && (after->kind == T_log_and || after->kind == T_log_or || - after->kind == T_question); +static var_t *read_global_compound_address(block_t *parent, + basic_block_t **bb, + global_constant_context_t *ctx) +{ + var_t return_decl; + func_t *callback = NULL; + lex_expect(T_open_bracket); + read_type_name_decl(GLOBAL_BLOCK, &return_decl); + type_t *type = return_decl.type; + int depth = return_decl.ptr_level; + if (type && abstract_function_pointer_follows()) { + int callback_level; + callback = + read_abstract_function_pointer(&return_decl, &callback_level); + if (callback_level != 1) + error_at("Incompatible compound literal address", cur_token_loc()); + } else if (type && type->func_signature && !type->is_direct_function_type && + !type->array_size && !depth) { + callback = type->func_signature; + } + lex_expect(T_close_bracket); + if (!type || type->array_size || (!callback && type->func_signature)) + error_at("Incompatible compound literal address", cur_token_loc()); + var_t declaration = return_decl; + declaration.ptr_level = callback && type->func_signature ? 0 : depth; + declaration.is_func = callback != NULL; + declaration.func_signature = callback; + validate_global_compound_address(ctx->dest, &declaration); + var_t *literal = name_var(require_typed_var(GLOBAL_BLOCK, type)); + literal->is_global = true; + literal->ptr_level = callback && type->func_signature ? 0 : depth; + literal->is_func = callback != NULL; + literal->func_signature = callback; + add_insn(GLOBAL_BLOCK, *bb, OP_allocat, literal, NULL, NULL, 0, NULL); + if (!callback && !depth && is_record_type(type)) { + parse_global_record_init(literal, GLOBAL_BLOCK); + } else { + lex_expect(T_open_curly); + if (lex_peek(T_close_curly, NULL)) + error_at("Scalar compound literal needs an initializer", + next_token_loc()); + read_global_assignment_var(literal); + lex_accept(T_comma); + lex_expect(T_close_curly); + } + var_t *address = + name_var(require_ref_var(parent, type, literal->ptr_level)); + address->is_global_address = true; + address->pointee_func_signature = callback; + add_insn(parent, *bb, OP_address_of, address, literal, NULL, 0, NULL); + return address; } -var_t *parse_global_constant_value(block_t *parent, basic_block_t **bb) +static var_t *read_global_function_cast_value(block_t *parent, + basic_block_t *bb, + block_t *scope, + func_t *signature) +{ + var_t *value = read_wide_global_literal_primary(parent, bb, scope); + value->type = signature->return_def.type; + value->ptr_level = 1; + value->func_signature = signature; + value->pointee_func_signature = NULL; + if (PTR_SIZE == 4) + value->init_val_hi = 0; + return value; +} + +static var_t *read_global_constant_primary(block_t *parent, + basic_block_t **bb, + global_constant_context_t *ctx) { var_t *val = NULL; - block_t *scope = initializer_name_scope(parent); + block_t *scope = ctx->scope; + bool scalar = ctx->dest != NULL; bool address_dereference = global_function_address_dereference_starts_here(); token_t *address_dereference_identifier = NULL; @@ -601,53 +680,65 @@ var_t *parse_global_constant_value(block_t *parent, basic_block_t **bb) */ func_t *cast_signature = read_global_function_pointer_cast(scope); - if (cast_signature) { - val = parse_global_constant_value(parent, bb); - if (val && val->is_func) - val->func_signature = cast_signature; - else if (val && !effective_pointer_depth(val)) { - /* An integer cast to a function pointer type is an explicit - * conversion: keep the cast's type on the value, so the element - * store does not take it for an implicit one. - */ - val->ptr_level = 1; - val->func_signature = cast_signature; - } - return val; - } + if (cast_signature) + return read_global_function_cast_value(parent, *bb, scope, + cast_signature); if (address_dereference) { address_dereference_identifier = consume_global_function_address_dereference(); - if (!find_visible_func(address_dereference_identifier->literal, scope)) + func_t *addressed = + find_visible_func(address_dereference_identifier->literal, scope); + if (!addressed) error_at("Function address requires a visible declaration", cur_token_loc()); - val = require_func_symbol_var(parent); - val->var_name = intern_string(address_dereference_identifier->literal); - val->is_func = true; + val = function_designator_value(parent, addressed); return val; } if (global_pointer_cast_starts_here(scope)) { int saved_stride = global_pointer_cast_stride; - func_t *slot_signature; - int stride = read_global_pointer_cast(scope, &slot_signature); + var_t cast; + int stride = read_global_pointer_cast(scope, &cast); /* In a chain of casts the outermost one decides the stride. */ if (saved_stride) stride = saved_stride; - if (!global_address_operand_starts_here(scope)) - return read_global_cast_integer_address(parent, *bb, scope, stride, - slot_signature); - global_pointer_cast_stride = stride; - val = parse_global_constant_value(parent, bb); - global_pointer_cast_stride = saved_stride; + if (!global_address_operand_starts_here(scope)) { + val = read_global_cast_integer_address(parent, *bb, scope, stride, + cast.pointee_func_signature); + } else { + global_pointer_cast_stride = stride; + val = read_global_constant_primary(parent, bb, ctx); + global_pointer_cast_stride = saved_stride; + } + if (val->is_function_designator) + error_at("incompatible function pointer types", cur_token_loc()); + val->address_stride = stride; + val->type = cast.type; + val->ptr_level = cast.ptr_level; + val->is_func = false; + val->func_signature = NULL; + val->pointee_func_signature = cast.pointee_func_signature; + val->is_const_qualified = cast.is_const_qualified; + val->is_const_pointer = cast.is_const_pointer; + val->pointer_const_mask = cast.pointer_const_mask; + val->is_volatile = cast.is_volatile; + val->pointer_volatile_mask = cast.pointer_volatile_mask; + val->is_void_null_pointer = is_plain_void_pointer(val) && + val->is_const && !val->init_val && + !val->init_val_hi; return val; } explicit_address = lex_accept(T_ampersand); + if (explicit_address && (scalar || global_constant_destination) && + scope == GLOBAL_BLOCK && global_compound_literal_starts_here()) { + global_constant_context_t compound = *ctx; + if (!compound.dest) + compound.dest = global_constant_destination; + return read_global_compound_address(parent, bb, &compound); + } if (explicit_address && subscripted_string_literal_starts_here()) return read_string_literal_element_address(parent, *bb, scope); - if (!explicit_address && string_address_offset_starts_here()) - return read_string_address_offset(parent, *bb, scope); if (grouped_global_function_designator_starts_here(false)) { lex_expect(T_open_bracket); @@ -662,13 +753,12 @@ var_t *parse_global_constant_value(block_t *parent, basic_block_t **bb) lex_peek(T_log_not, NULL) || lex_peek(T_open_bracket, NULL) || lex_peek(T_sizeof, NULL) || lex_peek(T_char, NULL) || lex_peek(T_wchar, NULL) || subscripted_string_literal_starts_here() || - global_tested_operand_starts_here(scope) || (lex_peek(T_identifier, constant_name) && find_scoped_constant(constant_name, scope)))) { /* Any integer constant expression, including casts, sizeof and grouped * subexpressions. The two-word reader keeps a wide value's high word. */ - val = read_wide_global_literal_expression(parent, *bb, scope); + val = read_wide_global_literal_primary(parent, *bb, scope); } else if (explicit_address || grouped_function_designator || lex_peek(T_identifier, NULL)) { char name[MAX_ID_LEN]; @@ -676,12 +766,10 @@ var_t *parse_global_constant_value(block_t *parent, basic_block_t **bb) if (lex_peek(T_identifier, name)) { func_t *func = find_visible_func(name, scope); if (func) { + val = function_designator_value(parent, func); lex_expect(T_identifier); if (grouped_function_designator) lex_expect(T_close_bracket); - val = require_func_symbol_var(parent); - val->var_name = intern_string(name); - val->is_func = true; return val; } if (explicit_address) { @@ -694,14 +782,8 @@ var_t *parse_global_constant_value(block_t *parent, basic_block_t **bb) if (object && object->is_global) { lex_expect(T_identifier); - val = require_ref_var(parent, object->type, - object->ptr_level); - val->var_name = gen_name(); - val->is_global_address = true; - add_insn(parent, *bb, OP_address_of, val, object, NULL, 0, - NULL); - return read_global_address_designator(scope, parent, bb, - &object, val, false); + return read_global_object_address(scope, parent, bb, object, + false); } } else { var_t *object = find_var(name, scope); @@ -715,30 +797,31 @@ var_t *parse_global_constant_value(block_t *parent, basic_block_t **bb) if (object && object->is_global && (subscripted ? object->array_dim2 : object->array_size)) { lex_expect(T_identifier); - val = require_ref_var(parent, object->type, - object->ptr_level); - val->var_name = gen_name(); - val->is_global_address = true; - add_insn(parent, *bb, OP_address_of, val, object, NULL, 0, - NULL); - return read_global_address_designator(scope, parent, bb, - &object, val, true); + return read_global_object_address(scope, parent, bb, object, + true); } } } if (explicit_address) error_at("Expected a global object or function after '&'", cur_token_loc()); - error_at("Global aggregate initializer requires a constant value", + if (scalar) + return read_wide_global_literal_primary(parent, *bb, scope); + error_at("Global initializer requires a constant value", cur_token_loc()); - } else if (lex_peek(T_string, NULL)) { + } else if (lex_peek(T_string, NULL) || lex_peek(T_wstring, NULL)) { /* A character-pointer member has the same constant-expression form as a * standalone global pointer: retain the rodata address, including * adjacent-literal concatenation, for the aggregate store. */ - read_literal_param(parent, *bb); + if (lex_peek(T_wstring, NULL)) + read_wstring_param(parent, *bb); + else + read_literal_param(parent, *bb); val = opstack_pop(); } else { + if (scalar) + return read_wide_global_literal_primary(parent, *bb, scope); error_at("Global array initialization requires constant values", next_token_loc()); } @@ -746,6 +829,28 @@ var_t *parse_global_constant_value(block_t *parent, basic_block_t **bb) return val; } +static var_t *read_global_constant_value(block_t *parent, + basic_block_t **bb, + global_constant_context_t *ctx) +{ + var_t *saved_destination = global_constant_destination; + global_constant_destination = ctx->dest; + var_t *first = read_global_constant_primary(parent, bb, ctx); + var_t *value = + read_wide_global_literal_tail(parent, *bb, ctx->scope, first); + global_constant_destination = saved_destination; + if (value->is_function_designator || value->is_func) + diagnose_global_function_value(ctx, value); + return value; +} + +var_t *parse_global_constant_value(block_t *parent, basic_block_t **bb) +{ + global_constant_context_t ctx = {0}; + ctx.scope = initializer_name_scope(parent); + return read_global_constant_value(parent, bb, &ctx); +} + bool is_record_type(const type_t *type) { /* A callback typedef whose function returns a record keeps that record's @@ -787,36 +892,35 @@ void parse_string_row_init(block_t *parent, var_t *target_addr, int start); -/* Store @zero, a loaded zero constant, into each of @size bytes at @addr. The - * stores are byte-wide so that a record or a wide element needs no store width - * a narrow backend cannot encode. +/* Store @zero into each byte at @addr, using @alignment to select the widest + * safe indirect store. */ void emit_zero_bytes(block_t *parent, basic_block_t **bb, var_t *addr, int size, + int alignment, var_t *zero) { - for (int offset = 0; offset < size; offset++) { - var_t *byte_addr = compute_element_address(parent, bb, addr, offset, 1); - add_insn(parent, *bb, OP_write, NULL, byte_addr, zero, 1, NULL); + for (int offset = 0; offset < size;) { + int width = byte_access_width(size - offset, alignment, offset); + var_t *part = compute_element_address(parent, bb, addr, offset, 1); + + add_insn(parent, *bb, OP_write, NULL, part, zero, width, NULL); + offset += width; } } var_t *emit_zero_constant(block_t *parent, basic_block_t **bb) { - var_t *zero = require_var(parent); - - zero->var_name = gen_name(); - zero->init_val = 0; - add_insn(parent, *bb, OP_load_constant, zero, NULL, NULL, 0, NULL); - return zero; + return load_constant(parent, *bb, 0, TY_int); } /* Store zero into every byte of elements [from, to) of the array at @base. */ void emit_zero_elements(block_t *parent, basic_block_t **bb, var_t *base, + const var_t *element, int from, int to, int elem_size) @@ -826,10 +930,20 @@ void emit_zero_elements(block_t *parent, if (from >= to) return; zero = emit_zero_constant(parent, bb); - for (int i = from; i < to; i++) - emit_zero_bytes(parent, bb, - compute_element_address(parent, bb, base, i, elem_size), - elem_size, zero); + for (int i = from; i < to; i++) { + var_t *address = + compute_element_address(parent, bb, base, i, elem_size); + + if (elem_size == 1 || + (elem_size <= 4 && element && element->type && + (element->is_func || effective_pointer_depth(element) || + !is_record_type(element->type)))) + add_insn(parent, *bb, OP_write, NULL, address, zero, elem_size, + NULL); + else + emit_zero_bytes(parent, bb, address, elem_size, + initializer_alignment(element), zero); + } } /* The width of one element of @array: a pointer for an array of pointers or of @@ -902,13 +1016,77 @@ bool accept_slot_designator(block_t *scope, int bound, int *slot) return true; } +static var_t *read_array_scalar_initializer(block_t *parent, + basic_block_t **bb, + bool allow_global_constant, + block_t *global_array_scope) +{ + if (parent == GLOBAL_BLOCK && allow_global_constant) { + char name[MAX_ID_LEN]; + + if (global_array_scope && lex_peek(T_identifier, name) && + !find_scoped_constant(name, global_array_scope) && + !find_visible_func(name, global_array_scope)) { + var_t *object = find_var(name, global_array_scope); + + if (!object || !object->is_global || !object->array_size) + error_at( + global_array_scope == GLOBAL_BLOCK + ? "Global array initialization requires constant values" + : "Identifier is not an integer constant", + next_token_loc()); + } + return parse_global_constant_value(parent, bb); + } + read_expr(parent, bb); + read_ternary_operation(parent, bb); + return opstack_pop(); +} + +static bool parse_array_element_init(block_t *parent, + basic_block_t **bb, + const var_t *field, + var_t *address, + int elem_size, + bool overwritten, + var_t **parsed_value) +{ + var_t *value = NULL; + bool comma_consumed = false; + + if (parsed_value) + *parsed_value = NULL; + + if (lex_peek(T_open_curly, NULL) && is_record_type(field->type)) { + type_t *record_type = resolve_record_type(field->type); + lex_expect(T_open_curly); + parse_struct_field_init(parent, bb, record_type, address); + lex_expect(T_close_curly); + } else if (unbraced_record_starts_here(field)) { + comma_consumed = parse_unbraced_record_init(parent, bb, field->type, + address, overwritten); + } else { + value = read_array_scalar_initializer(parent, bb, !parsed_value, NULL); + } + + if (value) { + if (parsed_value) + *parsed_value = value; + var_t *stored = value->is_func + ? value + : resize_to(parent, bb, value, field->type, + parsed_value ? 0 : field->ptr_level); + add_insn(parent, *bb, OP_write, NULL, address, stored, elem_size, NULL); + } + return comma_consumed; +} + /* Parse one row of a two-dimensional array. An unbraced row returns true when a * brace-elided record element consumed the comma after the row's last * initializer, so the enclosing list must not expect it again. Only a braced * row owns designators; in an unbraced one they belong to the enclosing list. - * As in the plane and hyperplane helpers below, `filled` is one past the - * highest slot written, so after designators in any order the zero fill covers - * exactly the slots nothing wrote. + * `filled` is one past the highest slot written, so after designators in any + * order the zero fill covers exactly the slots nothing wrote. */ bool parse_array_field_row_values(block_t *parent, basic_block_t **bb, @@ -937,7 +1115,6 @@ bool parse_array_field_row_values(block_t *parent, return false; } while (!lex_peek(T_close_curly, NULL)) { - var_t *value = NULL; var_t *elem_addr; if (braced) @@ -946,42 +1123,19 @@ bool parse_array_field_row_values(block_t *parent, if (count >= field->array_dim2) error_at("Too many elements in array initializer", next_token_loc()); - emit_zero_elements(parent, bb, target_addr, start + filled, + emit_zero_elements(parent, bb, target_addr, field, start + filled, start + count, elem_size); elem_addr = compute_element_address(parent, bb, target_addr, start + count, elem_size); - comma_consumed = false; - if (lex_peek(T_open_curly, NULL) && is_record_type(field->type)) { - type_t *record_type = resolve_record_type(field->type); - lex_expect(T_open_curly); - parse_struct_field_init(parent, bb, record_type, elem_addr); - lex_expect(T_close_curly); - } else if (unbraced_record_starts_here(field)) { - comma_consumed = parse_unbraced_record_init( - parent, bb, field->type, elem_addr, count < filled); - } else if (parent == GLOBAL_BLOCK) { - value = parse_global_constant_value(parent, bb); - } else { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - value = opstack_pop(); - } - - if (value) { - var_t *stored = value->is_func - ? value - : resize_to(parent, bb, value, field->type, - field->ptr_level); - add_insn(parent, *bb, OP_write, NULL, elem_addr, stored, elem_size, - NULL); - } + comma_consumed = parse_array_element_init( + parent, bb, field, elem_addr, elem_size, count < filled, NULL); count++; if (count > filled) filled = count; if (count == field->array_dim2) { - if (braced && (comma_consumed || lex_accept(T_comma)) && + if (braced && initializer_separator(comma_consumed) && !lex_peek(T_close_curly, NULL)) { if (lex_peek(T_open_square, NULL)) continue; @@ -990,7 +1144,7 @@ bool parse_array_field_row_values(block_t *parent, } break; } - if (!comma_consumed && !lex_accept(T_comma)) + if (!initializer_separator(comma_consumed)) break; comma_consumed = true; } @@ -999,58 +1153,62 @@ bool parse_array_field_row_values(block_t *parent, comma_consumed = false; } - emit_zero_elements(parent, bb, target_addr, start + filled, + emit_zero_elements(parent, bb, target_addr, field, start + filled, start + field->array_dim2, elem_size); return comma_consumed; } -void parse_array_field_row_init(block_t *parent, - basic_block_t **bb, - const var_t *field, - var_t *target_addr, - int start) -{ - parse_array_field_row_values(parent, bb, field, target_addr, start, true); -} - -/* Parse one braced plane of a three-dimensional array. Rows reuse the - * two-dimensional helper, which also supplies C99's trailing zero fill. +/* Parse nested array dimensions recursively; the row parser owns scalar + * elements and string rows, while each outer level handles its designators and + * zero-filled gaps. */ -void parse_array_field_plane_values(block_t *parent, - basic_block_t **bb, - const var_t *field, - var_t *target_addr, - int start, - bool braced) +static bool parse_nested_array_values(block_t *parent, + basic_block_t **bb, + const var_t *array, + var_t *target_addr, + int start, + bool braced) { - var_t row; - int rows = 0; + var_t child; + int count = 0; int filled = 0; - int row_width = field->array_dim3; - int elem_size = array_element_size(field); - - memcpy(&row, field, sizeof(row)); - row.array_dim2 = row_width; - row.array_dim3 = 0; + int stride; + int elem_size = array_element_size(array); + + if (!array->array_dim3) + return parse_array_field_row_values(parent, bb, array, target_addr, + start, braced); + memcpy(&child, array, sizeof(child)); + fixed_array_var_drop_outer(&child); + stride = fixed_array_inner_count(&child); if (braced) { lex_expect(T_open_curly); reject_empty_initializer_in_strict_c99(); } while (!lex_peek(T_close_curly, NULL)) { + bool child_comma; + accept_slot_designator(initializer_name_scope(parent), - field->array_dim2, &rows); - if (rows >= field->array_dim2) - error_at("Too many rows in array initializer", next_token_loc()); - emit_zero_elements(parent, bb, target_addr, start + filled * row_width, - start + rows * row_width, elem_size); - bool row_comma = parse_array_field_row_values( - parent, bb, &row, target_addr, start + rows * row_width, - lex_peek(T_open_curly, NULL)); - rows++; - if (rows > filled) - filled = rows; - if (rows == field->array_dim2) { - if (braced && (row_comma || lex_accept(T_comma)) && + array->array_dim2, &count); + if (count >= array->array_dim2) + error_at(array->array_dim4 ? "Too many planes in array initializer" + : "Too many rows in array initializer", + next_token_loc()); + emit_zero_elements(parent, bb, target_addr, array, + start + filled * stride, start + count * stride, + elem_size); + child_comma = parse_nested_array_values(parent, bb, &child, target_addr, + start + count * stride, + lex_peek(T_open_curly, NULL)); + count++; + if (count > filled) + filled = count; + if (array->array_dim4) { + if (!lex_accept(T_comma)) + break; + continue; + } else if (count == array->array_dim2) { + if (braced && initializer_separator(child_comma) && !lex_peek(T_close_curly, NULL)) { if (lex_peek(T_open_square, NULL)) continue; @@ -1058,69 +1216,27 @@ void parse_array_field_plane_values(block_t *parent, next_token_loc()); } break; - } - if (!row_comma && !lex_accept(T_comma)) + } else if (!initializer_separator(child_comma)) { break; + } } if (braced) lex_expect(T_close_curly); - - emit_zero_elements(parent, bb, target_addr, start + filled * row_width, - start + field->array_dim2 * row_width, elem_size); + emit_zero_elements(parent, bb, target_addr, array, start + filled * stride, + start + array->array_dim2 * stride, elem_size); + return false; } -void parse_array_field_plane_init(block_t *parent, - basic_block_t **bb, - const var_t *field, - var_t *target_addr, - int start) +static int parse_nested_array_initializer(block_t *parent, + basic_block_t **bb, + const var_t *array, + var_t *target_addr, + int start) { - parse_array_field_plane_values(parent, bb, field, target_addr, start, true); -} - -/* Parse one braced outer slice of a four-dimensional array. Each contained - * three-dimensional plane reuses the existing plane parser, so row bounds and - * trailing zero fill remain identical at every nesting level. - */ -void parse_array_field_hyperplane_init(block_t *parent, - basic_block_t **bb, - const var_t *field, - var_t *target_addr, - int start) -{ - var_t plane; - int planes = 0; - int filled = 0; - int plane_size = field->array_dim3 * field->array_dim4; - int elem_size = array_element_size(field); - - memcpy(&plane, field, sizeof(plane)); - plane.array_size = plane_size; - plane.array_dim2 = field->array_dim3; - plane.array_dim3 = field->array_dim4; - plane.array_dim4 = 0; - lex_expect(T_open_curly); - reject_empty_initializer_in_strict_c99(); - while (!lex_peek(T_close_curly, NULL)) { - accept_slot_designator(initializer_name_scope(parent), - field->array_dim2, &planes); - if (planes >= field->array_dim2) - error_at("Too many planes in array initializer", next_token_loc()); - emit_zero_elements(parent, bb, target_addr, start + filled * plane_size, - start + planes * plane_size, elem_size); - parse_array_field_plane_values(parent, bb, &plane, target_addr, - start + planes * plane_size, - lex_peek(T_open_curly, NULL)); - planes++; - if (planes > filled) - filled = planes; - if (!lex_accept(T_comma)) - break; - } - lex_expect(T_close_curly); - - emit_zero_elements(parent, bb, target_addr, start + filled * plane_size, - start + field->array_dim2 * plane_size, elem_size); + if (!array->array_dim2 || !lex_peek(T_open_curly, NULL)) + return 0; + parse_nested_array_values(parent, bb, array, target_addr, start, true); + return fixed_array_inner_count(array); } /* Read a string literal and every adjacent one after it, decoded and joined @@ -1152,51 +1268,122 @@ int read_concatenated_string(char *combined) return used; } -/* An array member can be initialized directly by a string literal just like a - * standalone character array. The member has no independent var_t storage, so - * write through its already-computed field address rather than routing this - * through parse_string_array_init(). - */ -void parse_string_field_init(block_t *parent, - basic_block_t **bb, - const var_t *field, - var_t *target_addr) +static void emit_string_array_units(block_t *parent, + basic_block_t **bb, + const var_t *array, + var_t *base, + int count, + const void *units, + int initialized, + bool wide) { - char combined[MAX_STRING_LEN]; - int len; - - /* The terminating null is dropped when only the characters fit (C99 - * 6.7.8p14). - */ - len = read_concatenated_string(combined) + 1; - if (len - 1 > field->array_size) - error_at("String initializer is too long for character array", - cur_token_loc()); + int size = array->type->size; - for (int i = 0; i < field->array_size; i++) { - var_t *value = require_var(parent); - var_t *addr = compute_element_address(parent, bb, target_addr, i, 1); + for (int i = 0; i < count; i++) { + var_t *value = + wide ? require_typed_var(parent, array->type) : require_var(parent); + var_t *address; value->var_name = gen_name(); - value->init_val = i < len ? (unsigned char) combined[i] : 0; + value->init_val = i >= initialized ? 0 + : wide ? ((const int *) units)[i] + : ((const unsigned char *) units)[i]; value->is_const = true; add_insn(parent, *bb, OP_load_constant, value, NULL, NULL, 0, NULL); - add_insn(parent, *bb, OP_write, NULL, addr, value, 1, NULL); + address = compute_element_address(parent, bb, base, i, size); + add_insn(parent, *bb, OP_write, NULL, address, value, size, NULL); } } +static void initialize_string_array(const var_t *array, + block_t *parent, + basic_block_t **bb, + var_t *base, + const void *units, + int length, + bool wide, + var_t *infer_bound) +{ + /* Both decoders omit the terminator. Narrow arrays initialize it when it + * fits; wide arrays leave its zero-filled unit to the array storage. + */ + int inferred_size = length + 1; + int initialized = wide ? length : inferred_size; + int array_size = array->array_size; + + if (infer_bound && infer_bound->has_unsized_array) { + array_size = inferred_size; + infer_bound->array_size = array_size; + infer_bound->has_unsized_array = false; + } else if (length > array_size) + error_at(wide ? "Wide string initializer is too long for array" + : "String initializer is too long for character array", + cur_token_loc()); + + if (infer_bound && !wide && initialized > array_size) + initialized = array_size; + int count = infer_bound && !wide && parent == GLOBAL_BLOCK ? initialized + : array_size; + emit_string_array_units(parent, bb, array, base, count, units, initialized, + wide); +} + +/* Decode and lower a string initializer; target_addr may be a field address, + * while infer_bound is set only for a declared array with unknown size. + */ +static void parse_string_array_init_to(const var_t *array, + block_t *parent, + basic_block_t **bb, + var_t *target_addr, + var_t *infer_bound, + bool wide) +{ + union { + char narrow[MAX_STRING_LEN]; + int wide[MAX_STRING_LEN]; + } units; + int length = wide ? read_wstring_units(units.wide, MAX_STRING_LEN) + : read_concatenated_string(units.narrow); + + initialize_string_array( + array, parent, bb, target_addr, + wide ? (const void *) units.wide : (const void *) units.narrow, length, + wide, infer_bound); +} + +/* A member has no independent var_t storage, so use its computed address. */ +void parse_string_field_init(block_t *parent, + basic_block_t **bb, + const var_t *field, + var_t *target_addr) +{ + parse_string_array_init_to(field, parent, bb, target_addr, NULL, false); +} + bool is_char_array(const var_t *var) { return var && !var->ptr_level && !var->array_dim2 && !var->array_dim3 && !var->array_dim4 && - compatible_decl_type(var->type, find_type("char", true)); + compatible_decl_type(var->type, find_builtin_type("char")); } bool is_wchar_array(const var_t *var) { return var && !var->ptr_level && !var->array_dim2 && !var->array_dim3 && !var->array_dim4 && - compatible_decl_type(var->type, find_type("wchar_t", true)); + compatible_decl_type(var->type, find_builtin_type("wchar_t")); +} + +static void parse_braced_string_array_init(var_t *var, + block_t *parent, + basic_block_t **bb) +{ + if (lex_peek(T_wstring, NULL)) + parse_wstring_array_init(var, parent, bb); + else + parse_string_array_init(var, parent, bb); + lex_accept(T_comma); + lex_expect(T_close_curly); } /* C99 permits a string literal initializer only for an array of matching @@ -1221,26 +1408,7 @@ void parse_wstring_field_init(block_t *parent, const var_t *field, var_t *target_addr) { - int values[MAX_STRING_LEN]; - int length; - - length = read_wstring_units(values, MAX_STRING_LEN); - if (length > field->array_size) - error_at("Wide string initializer is too long for array", - cur_token_loc()); - - for (int i = 0; i < field->array_size; i++) { - var_t *value = require_typed_var(parent, field->type); - var_t *addr = compute_element_address(parent, bb, target_addr, i, - field->type->size); - - value->var_name = gen_name(); - value->init_val = i < length ? values[i] : 0; - value->is_const = true; - add_insn(parent, *bb, OP_load_constant, value, NULL, NULL, 0, NULL); - add_insn(parent, *bb, OP_write, NULL, addr, value, field->type->size, - NULL); - } + parse_string_array_init_to(field, parent, bb, target_addr, NULL, true); } /* A string literal also initializes one innermost row of a multidimensional @@ -1264,9 +1432,9 @@ bool string_row_starts_here(const var_t *array) (after->next->kind != T_comma && after->next->kind != T_close_curly)) return false; if ((kind == T_string && - !compatible_decl_type(array->type, find_type("char", true))) || + !compatible_decl_type(array->type, find_builtin_type("char"))) || (kind == T_wstring && - !compatible_decl_type(array->type, find_type("wchar_t", true)))) + !compatible_decl_type(array->type, find_builtin_type("wchar_t")))) error_at( "String literal initializer has incompatible array element type", next_token_loc()); @@ -1314,91 +1482,12 @@ void parse_array_field_init(block_t *parent, const var_t *field, var_t *target_addr) { - int count = 0; - - /* One past the highest element written so far. A designator can move - * backwards, so zero fill starts here rather than at `count`, which would - * clear rows that an earlier designator already stored. - */ - int filled = 0; - int elem_size = array_element_size(field); - - lex_expect(T_open_curly); - reject_empty_initializer_in_strict_c99(); - while (!lex_peek(T_close_curly, NULL)) { - var_t *value = NULL; - var_t slice; - bool comma_consumed = false; - - /* `slice` is the array whose one element the next initializer fills. - * Without a designator that is the member itself. Each subscript of a - * designator such as `[1] = { ... }` or `[1][0] = 3` descends one - * level, so the braced row, plane and scalar paths below see exactly - * the shape the designator names and reordered rows stay valid. - */ - memcpy(&slice, field, sizeof(slice)); - if (read_array_designator(initializer_name_scope(parent), field, 0, - false, &count, &slice)) - lex_expect(T_assign); - - if (count >= field->array_size) - error_at("Too many elements in array initializer", - next_token_loc()); - emit_zero_elements(parent, bb, target_addr, filled, count, elem_size); - - var_t *elem_addr = - compute_element_address(parent, bb, target_addr, count, elem_size); - if (slice.array_dim4 && lex_peek(T_open_curly, NULL)) { - parse_array_field_hyperplane_init(parent, bb, &slice, target_addr, - count); - count += fixed_array_inner_count(&slice); - } else if (slice.array_dim3 && lex_peek(T_open_curly, NULL)) { - parse_array_field_plane_init(parent, bb, &slice, target_addr, - count); - count += fixed_array_inner_count(&slice); - } else if (slice.array_dim2 && lex_peek(T_open_curly, NULL)) { - parse_array_field_row_init(parent, bb, &slice, target_addr, count); - count += fixed_array_inner_count(&slice); - } else if (string_row_starts_here(&slice)) { - parse_string_row_init(parent, bb, &slice, target_addr, count); - count += string_row_width(&slice); - } else { - if (lex_peek(T_open_curly, NULL) && is_record_type(field->type)) { - type_t *record_type = resolve_record_type(field->type); - lex_expect(T_open_curly); - parse_struct_field_init(parent, bb, record_type, elem_addr); - lex_expect(T_close_curly); - } else if (unbraced_record_starts_here(field)) { - comma_consumed = parse_unbraced_record_init( - parent, bb, field->type, elem_addr, count < filled); - } else if (parent == GLOBAL_BLOCK) { - value = parse_global_constant_value(parent, bb); - } else { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - value = opstack_pop(); - } + var_t array; - if (value) { - var_t *stored = value->is_func - ? value - : resize_to(parent, bb, value, field->type, - field->ptr_level); - add_insn(parent, *bb, OP_write, NULL, elem_addr, stored, - elem_size, NULL); - } - count++; - } - - if (count > filled) - filled = count; - if (!comma_consumed && !lex_accept(T_comma)) - break; - } - lex_expect(T_close_curly); - - emit_zero_elements(parent, bb, target_addr, filled, field->array_size, - elem_size); + memcpy(&array, field, sizeof(array)); + if (parent == GLOBAL_BLOCK) + array.scope = initializer_name_scope(parent); + parse_array_init_at(&array, parent, bb, target_addr, true); } /* Initialize the record at @addr from @value, an initializer already read that @@ -1585,14 +1674,15 @@ bool parse_struct_field_values(block_t *parent, var_t *zero = emit_zero_constant(parent, bb); if (is_union) - emit_zero_bytes(parent, bb, target_addr, struct_type->size, zero); + emit_zero_bytes(parent, bb, target_addr, struct_type->size, + alignment_type(struct_type), zero); for (int i = 0; !is_union && i < struct_type->num_fields; i++) { var_t *field = &struct_type->fields[i]; emit_zero_bytes( parent, bb, compute_field_address(parent, bb, target_addr, field), - size_var(field), zero); + size_var(field), initializer_alignment(field), zero); } } @@ -1801,7 +1891,7 @@ bool parse_struct_field_values(block_t *parent, member_address(parent, bb, target_addr, field, field_addr)); } else if (field && field->array_size && !field->ptr_level && compatible_decl_type(field->type, - find_type("wchar_t", true)) && + find_builtin_type("wchar_t")) && lex_peek(T_wstring, NULL)) { parse_wstring_field_init( parent, bb, field, @@ -1825,12 +1915,9 @@ bool parse_struct_field_values(block_t *parent, parent, bb, field->type, member_address(parent, bb, target_addr, field, field_addr), true); - } else if (parent == GLOBAL_BLOCK) { - field_val_raw = parse_global_constant_value(parent, bb); } else { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - field_val_raw = opstack_pop(); + field_val_raw = + read_array_scalar_initializer(parent, bb, true, NULL); } if (field_val_raw) { @@ -1917,7 +2004,7 @@ bool parse_struct_field_values(block_t *parent, break; } } - if (!nested_comma_consumed && !lex_accept(T_comma)) + if (!initializer_separator(nested_comma_consumed)) break; comma_consumed = true; if (lex_peek(T_close_curly, NULL)) @@ -1942,26 +2029,21 @@ bool parse_struct_field_values(block_t *parent, void parse_array_literal_expr(block_t *parent, basic_block_t **bb) { - var_t *array_var = require_var(parent); - array_var->var_name = gen_name(); + var_t *array_var = require_named_var(parent); array_var->is_compound_literal = true; int element_count = 0; var_t *first_element = NULL; if (!lex_peek(T_close_curly, NULL)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - first_element = opstack_pop(); + first_element = read_array_scalar_initializer(parent, bb, false, NULL); element_count = 1; while (lex_accept(T_comma)) { if (lex_peek(T_close_curly, NULL)) break; - read_expr(parent, bb); - read_ternary_operation(parent, bb); - opstack_pop(); + read_array_scalar_initializer(parent, bb, false, NULL); element_count++; } } @@ -2000,17 +2082,7 @@ basic_block_t *handle_return_statement(block_t *parent, basic_block_t *bb) error_at("void function cannot return an expression in C99", cur_token_loc()); - if (!read_assignment_expression(parent, &bb)) { - read_expr(parent, &bb); - read_ternary_operation(parent, &bb); - } - while (lex_accept(T_comma)) { - discard_operand(parent, bb); - if (!read_assignment_expression(parent, &bb)) { - read_expr(parent, &bb); - read_ternary_operation(parent, &bb); - } - } + read_comma_expression(parent, &bb, false); lex_expect(T_semicolon); var_t *rs1 = opstack_pop(); @@ -2024,10 +2096,8 @@ basic_block_t *handle_return_statement(block_t *parent, basic_block_t *bb) cur_token_loc()); if (rs1 && rs1->array_size > 0 && rs1->var_name[0] == '.' && !strict_c99) { - var_t *val = require_var(parent); - val->type = rs1->type; + var_t *val = name_var(require_typed_var(parent, rs1->type)); val->init_val = rs1->init_val; - val->var_name = gen_name(); add_insn(parent, bb, OP_load_constant, val, NULL, NULL, 0, NULL); rs1 = val; } @@ -2068,19 +2138,29 @@ basic_block_t *handle_return_statement(block_t *parent, basic_block_t *bb) return NULL; } +static void emit_control_branch(block_t *parent, basic_block_t *bb) +{ + var_t *condition = opstack_pop(); + + reject_record_operand(condition); + add_insn(parent, bb, OP_branch, NULL, condition, NULL, 0, NULL); +} + +static void read_control_branch(block_t *parent, basic_block_t **bb) +{ + lex_expect(T_open_bracket); + read_control_expression(parent, bb); + lex_expect(T_close_bracket); + emit_control_branch(parent, *bb); +} + basic_block_t *handle_if_statement(block_t *parent, basic_block_t *bb) { basic_block_t *n = bb_create(parent); bb_connect(bb, n, NEXT); bb = n; - lex_expect(T_open_bracket); - read_control_expression(parent, &bb); - lex_expect(T_close_bracket); - - var_t *vd = opstack_pop(); - reject_record_operand(vd); - add_insn(parent, bb, OP_branch, NULL, vd, NULL, 0, NULL); + read_control_branch(parent, &bb); basic_block_t *then_ = bb_create(parent); basic_block_t *else_ = bb_create(parent); @@ -2088,35 +2168,14 @@ basic_block_t *handle_if_statement(block_t *parent, basic_block_t *bb) bb_connect(bb, else_, ELSE); basic_block_t *then_body = read_body_statement(parent, then_); - basic_block_t *then_next_ = NULL; - if (then_body) { - then_next_ = bb_create(parent); - bb_connect(then_body, then_next_, NEXT); - } - - if (lex_accept(T_else)) { - basic_block_t *else_body = read_body_statement(parent, else_); - basic_block_t *else_next_ = NULL; - if (else_body) { - else_next_ = bb_create(parent); - bb_connect(else_body, else_next_, NEXT); - } - - if (then_next_ && else_next_) { - basic_block_t *next_ = bb_create(parent); - bb_connect(then_next_, next_, NEXT); - bb_connect(else_next_, next_, NEXT); - return next_; - } - - return then_next_ ? then_next_ : else_next_; - } else { - if (then_next_) { - bb_connect(else_, then_next_, NEXT); - return then_next_; - } - return else_; - } + basic_block_t *else_body = + lex_accept(T_else) ? read_body_statement(parent, else_) : else_; + if (!then_body && !else_body) + return NULL; + basic_block_t *next = bb_create(parent); + bb_connect(then_body, next, NEXT); + bb_connect(else_body, next, NEXT); + return next; } basic_block_t *handle_while_statement(block_t *parent, basic_block_t *bb) @@ -2128,13 +2187,7 @@ basic_block_t *handle_while_statement(block_t *parent, basic_block_t *bb) continue_bb_push(bb); basic_block_t *cond = bb; - lex_expect(T_open_bracket); - read_control_expression(parent, &bb); - lex_expect(T_close_bracket); - - var_t *vd = opstack_pop(); - reject_record_operand(vd); - add_insn(parent, bb, OP_branch, NULL, vd, NULL, 0, NULL); + read_control_branch(parent, &bb); basic_block_t *then_ = bb_create(parent); basic_block_t *else_ = bb_create(parent); @@ -2153,54 +2206,25 @@ basic_block_t *handle_while_statement(block_t *parent, basic_block_t *bb) return else_; } -basic_block_t *handle_goto_statement(block_t *parent, basic_block_t *bb) +basic_block_t *handle_goto_statement(basic_block_t *bb) { - /* Since a goto splits the current program into two basic blocks and makes - * the subsequent basic block unreachable, this causes problems for later - * CFG operations. Therefore, we create a fake if that always executes to - * wrap the goto, and connect the unreachable basic block to the else - * branch. Finally, return this else block. - * - * after: a = b + c; goto label; c *= d; - * - * before: a = b + c; if (1) - * goto label; - * c *= d; - */ - char token[MAX_ID_LEN]; if (!lex_peek(T_identifier, token)) error_at("Expected identifier after 'goto'", next_token_loc()); - lex_expect(T_identifier); lex_expect(T_semicolon); - basic_block_t *fake_if = bb_create(parent); - bb_connect(bb, fake_if, NEXT); - var_t *val = require_var(parent); - val->var_name = gen_name(); - val->init_val = 1; - add_insn(parent, fake_if, OP_load_constant, val, NULL, NULL, 0, NULL); - add_insn(parent, fake_if, OP_branch, NULL, val, NULL, 0, NULL); - - basic_block_t *then_ = bb_create(parent); - basic_block_t *else_ = bb_create(parent); - bb_connect(fake_if, then_, THEN); - bb_connect(fake_if, else_, ELSE); - - add_insn(parent, then_, OP_jump, NULL, NULL, NULL, 0, token); label_t *label = find_label(token); if (label) { label->used = true; - bb_connect(then_, label->bb, NEXT); - return else_; + bb_connect(bb, label->bb, NEXT); + } else { + if (backpatch_bb_idx >= MAX_LABELS) + error_at("Too many forward-referenced labels", cur_token_loc()); + backpatch_label[backpatch_bb_idx] = intern_string(token); + backpatch_bb[backpatch_bb_idx++] = bb; } - - if (backpatch_bb_idx > MAX_LABELS - 1) - error_at("Too many forward-referenced labels", cur_token_loc()); - - backpatch_bb[backpatch_bb_idx++] = then_; - return else_; + return NULL; } int read_const_expr(block_t *scope); @@ -2247,9 +2271,9 @@ static void reject_string_element_initializer(const var_t *var) (token->next->kind != T_comma && token->next->kind != T_close_curly)) return; if ((kind == T_string && - compatible_decl_type(var->type, find_type("char", true))) || + compatible_decl_type(var->type, find_builtin_type("char"))) || (kind == T_wstring && - compatible_decl_type(var->type, find_type("wchar_t", true)))) + compatible_decl_type(var->type, find_builtin_type("wchar_t")))) error_at("String literal cannot initialize a single array element", next_token_loc()); error_at("String literal initializer has incompatible array element type", @@ -2257,22 +2281,33 @@ static void reject_string_element_initializer(const var_t *var) } void parse_array_init(var_t *var, block_t *parent, basic_block_t **bb) +{ + parse_array_init_at(var, parent, bb, var, false); +} + +static void parse_array_init_at(var_t *var, + block_t *parent, + basic_block_t **bb, + var_t *base_addr, + bool sparse_zero) { int count = 0; int inferred_size = 0; - var_t *base_addr = NULL; + int initialized_end = 0; /* An omitted outer bound of a multidimensional declaration already has an * inner-dimension product in `array_size`. It is nevertheless inferred from * the initializer, just like a one-dimensional `int a[]`. */ bool is_implicit = (var->array_size == 0 || var->has_unsized_array); + bool runtime_initializer = + parent != GLOBAL_BLOCK || var->is_compound_literal; block_t *initializer_scope = parent; if (parent == GLOBAL_BLOCK && var->scope && var->scope != GLOBAL_BLOCK) initializer_scope = var->scope; block_t *saved_initializer_scope = global_constant_initializer_scope; - if (parent == GLOBAL_BLOCK) + if (!runtime_initializer) global_constant_initializer_scope = initializer_scope; /* Elements of a pointer array are pointer-sized. Using the base type's @@ -2289,25 +2324,17 @@ void parse_array_init(var_t *var, block_t *parent, basic_block_t **bb) if (lex_peek(T_open_curly, NULL) && string_ends_braced_initializer(var, cur_token->next->next)) { lex_expect(T_open_curly); - if (lex_peek(T_wstring, NULL)) - parse_wstring_array_init(var, parent, bb); - else - parse_string_array_init(var, parent, bb); - lex_accept(T_comma); - lex_expect(T_close_curly); + parse_braced_string_array_init(var, parent, bb); global_constant_initializer_scope = saved_initializer_scope; return; } - base_addr = var; - - /* Reordered array designators can leave holes both before and after a - * written element. Initialize the whole automatic array first, then let - * explicit elements overwrite their slots. Byte stores also cover record - * elements without relying on a backend-wide aggregate store. + /* Reordered designators can leave holes before or after written elements. + * Ordinary automatic arrays are cleared up front; field arrays clear only + * skipped ranges. */ - if (parent != GLOBAL_BLOCK && !is_implicit) - emit_zero_elements(parent, bb, base_addr, 0, var->array_size, + if (runtime_initializer && !is_implicit && !sparse_zero) + emit_zero_elements(parent, bb, base_addr, var, 0, var->array_size, elem_size); lex_expect(T_open_curly); @@ -2344,41 +2371,19 @@ void parse_array_init(var_t *var, block_t *parent, basic_block_t **bb) error_at("Too many elements in array initializer", next_token_loc()); - /* A forward designator leaves a gap. Explicit arrays were zeroed - * before parsing and global storage begins zeroed, but an inferred - * local array has no known bound yet: zero the elements skipped - * past the highest one written, whatever form the next initializer - * takes. Every element below `inferred_size` is then either zero or - * written, which a brace-elided record relies on below. - */ - if (is_implicit && parent != GLOBAL_BLOCK) - emit_zero_elements(parent, bb, base_addr, inferred_size, count, - elem_size); - - if (initializer_var->array_dim4 && lex_peek(T_open_curly, NULL)) { - parse_array_field_hyperplane_init(parent, bb, initializer_var, - base_addr, count); - count += fixed_array_inner_count(initializer_var); - if (is_implicit && count > inferred_size) - inferred_size = count; - if (!lex_accept(T_comma)) - break; - continue; - } else if (initializer_var->array_dim3 && - lex_peek(T_open_curly, NULL)) { - parse_array_field_plane_init(parent, bb, initializer_var, - base_addr, count); - count += fixed_array_inner_count(initializer_var); - if (is_implicit && count > inferred_size) - inferred_size = count; - if (!lex_accept(T_comma)) - break; - continue; - } else if (initializer_var->array_dim2 && - lex_peek(T_open_curly, NULL)) { - parse_array_field_row_init(parent, bb, initializer_var, - base_addr, count); - count += fixed_array_inner_count(initializer_var); + /* Zero any skipped range from the highest initialized slot. */ + if (runtime_initializer && (is_implicit || sparse_zero)) + emit_zero_elements( + parent, bb, base_addr, var, + is_implicit ? inferred_size : initialized_end, count, + elem_size); + + int nested = parse_nested_array_initializer( + parent, bb, initializer_var, base_addr, count); + if (nested) { + count += nested; + if (count > initialized_end) + initialized_end = count; if (is_implicit && count > inferred_size) inferred_size = count; if (!lex_accept(T_comma)) @@ -2388,6 +2393,8 @@ void parse_array_init(var_t *var, block_t *parent, basic_block_t **bb) parse_string_row_init(parent, bb, initializer_var, base_addr, count); count += string_row_width(initializer_var); + if (count > initialized_end) + initialized_end = count; if (is_implicit && count > inferred_size) inferred_size = count; if (!lex_accept(T_comma) || lex_peek(T_close_curly, NULL)) @@ -2403,192 +2410,31 @@ void parse_array_init(var_t *var, block_t *parent, basic_block_t **bb) var_t *elem_addr = compute_element_address( parent, bb, base_addr, count, elem_size); - if (is_implicit && parent != GLOBAL_BLOCK && - count >= inferred_size) - emit_zero_elements(parent, bb, base_addr, count, count + 1, - elem_size); + if (runtime_initializer && (is_implicit || sparse_zero) && + count >= (is_implicit ? inferred_size : initialized_end)) + emit_zero_elements(parent, bb, base_addr, var, count, + count + 1, elem_size); elided_comma = parse_struct_field_values( parent, bb, resolve_record_type(var->type), elem_addr, true, true, NULL); - } else if (lex_peek(T_open_curly, NULL) && - is_record_type(var->type)) { - type_t *struct_type = resolve_record_type(var->type); - - var_t *elem_addr = compute_element_address( - parent, bb, base_addr, count, elem_size); - lex_expect(T_open_curly); - parse_struct_field_init(parent, bb, struct_type, elem_addr); - lex_expect(T_close_curly); - } else if (unbraced_record_starts_here(var)) { - /* Without braces a record element takes one initializer per - * member, so `struct p a[2] = { 1, 2, 3, 4 }` fills a[0] from 1 - * and 2 before a[1] starts. The record parser stops after its - * last member and leaves the comma to this loop. + } else if (is_record_type(var->type) && + (lex_peek(T_open_curly, NULL) || + unbraced_record_starts_here(var))) { + /* The shared element parser handles braced records and + * brace-elided member lists. It leaves a consumed comma for + * this loop and preserves members written by earlier + * designators. */ var_t *elem_addr = compute_element_address( parent, bb, base_addr, count, elem_size); - - /* An element already reached is zero or written, and may hold - * members a designator stored; a new one clears itself. - */ - elided_comma = parse_unbraced_record_init( - parent, bb, var->type, elem_addr, - !is_implicit || count < inferred_size); + elided_comma = parse_array_element_init( + parent, bb, var, elem_addr, elem_size, + (!is_implicit && !sparse_zero) || count < initialized_end, + NULL); } else { - /* A global initializer is restricted to simple constants, but - * it still has to be stored. Consuming the tokens and dropping - * the value left every global array zero-filled, while the same - * initializer on a local worked. - */ reject_string_element_initializer(var); - char leading_name[MAX_ID_LEN]; - - /* An arithmetic element, such as 1 ? 2 : 3 or 1 && 2, is folded - * as a constant. The runtime expression reader would branch, - * moving the global setup entry block to the branch's join and - * dropping every store emitted before it. - */ - if (parent == GLOBAL_BLOCK && - (lex_peek(T_numeric, NULL) || lex_peek(T_minus, NULL) || - global_tested_operand_starts_here(initializer_scope) || - lex_peek(T_char, NULL) || lex_peek(T_wchar, NULL) || - (lex_peek(T_identifier, leading_name) && - find_scoped_constant(leading_name, initializer_scope)) || - lex_peek(T_ampersand, NULL) || - grouped_global_function_designator_starts_here(true) || - global_function_address_dereference_starts_here() || - lex_peek(T_open_bracket, NULL) || - lex_peek(T_sizeof, NULL) || lex_peek(T_plus, NULL) || - lex_peek(T_bit_not, NULL) || lex_peek(T_log_not, NULL) || - string_element_appears_before_initializer_end( - cur_token->next) || - string_address_offset_starts_here())) { - /* Addresses, casts, sizeof and grouped or unary constant - * expressions share the reader of aggregate members. - */ - val = parse_global_constant_value(parent, bb); - } else { - char initializer_name[MAX_ID_LEN]; - char global_token[MAX_ID_LEN]; - bool function_initializer = - lex_peek(T_identifier, initializer_name) && - find_visible_func(initializer_name, initializer_scope); - var_t *object_constant = NULL; - - if (parent == GLOBAL_BLOCK && - lex_peek(T_identifier, global_token)) - object_constant = - find_var(global_token, initializer_scope); - - /* A static array named by a block-scope static's element is - * an address constant (C99 6.6p7), not an integer one, and - * takes the array-to-pointer path below. - */ - if (parent == GLOBAL_BLOCK && - initializer_scope != GLOBAL_BLOCK && - !lex_peek(T_string, NULL) && !function_initializer && - !lex_peek(T_ampersand, NULL) && - !(object_constant && object_constant->is_global && - object_constant->array_size)) { - /* Storage for a block-scope static lives globally, - * while its initializer is an integer constant - * expression in the surrounding block. Resolve local - * enumerators before emitting the global setup-store - * value. - */ - if (typed_global_literal_appears_before_initializer_end( - cur_token->next) || - string_element_appears_before_initializer_end( - cur_token->next)) { - /* read_const_expr() evaluates in an int, which - * drops the high word of a wide element. - */ - val = read_wide_global_literal_expression( - GLOBAL_BLOCK, *bb, var->scope); - } else { - val = require_var(GLOBAL_BLOCK); - val->var_name = gen_name(); - val->init_val = read_const_expr(var->scope); - val->is_const = true; - add_insn(GLOBAL_BLOCK, *bb, OP_load_constant, val, - NULL, NULL, 0, NULL); - } - } else { - if (parent == GLOBAL_BLOCK) { - char token[MAX_ID_LEN]; - bool enum_constant = - lex_peek(T_identifier, token) && - find_scoped_constant(token, parent); - bool function_constant = - lex_peek(T_identifier, token) && - find_visible_func(token, initializer_scope); - - if (!lex_peek(T_numeric, NULL) && - !lex_peek(T_minus, NULL) && - !lex_peek(T_string, NULL) && - !lex_peek(T_char, NULL) && - !lex_peek(T_wchar, NULL) && !enum_constant && - !function_constant && - !lex_peek(T_ampersand, NULL) && - !(object_constant && - object_constant->is_global && - object_constant->array_size)) - error_at( - "Global array initialization requires " - "constant " - "values", - next_token_loc()); - } - - if (parent == GLOBAL_BLOCK && object_constant && - object_constant->is_global && - object_constant->array_size) { - int stride = array_element_size(object_constant); - - /* Array-to-pointer conversion is a permitted - * address constant in static aggregate - * initializers. Preserve its row stride here rather - * than reading an object value, which global setup - * cannot do before GP is established. - */ - val = require_ref_var(parent, object_constant->type, - object_constant->ptr_level); - val->var_name = gen_name(); - lex_ident(T_identifier, global_token); - add_insn(parent, *bb, OP_address_of, val, - object_constant, NULL, 0, NULL); - if (object_constant->array_dim2) - stride *= object_constant->array_dim2; - if (object_constant->array_dim3) - stride *= object_constant->array_dim3; - if (object_constant->array_dim4) - stride *= object_constant->array_dim4; - if (object_constant->array_dim2 && - lex_peek(T_open_square, NULL)) { - var_t *row = object_constant; - - val->is_global_address = true; - val = read_global_address_designator( - initializer_scope, parent, bb, &row, val, - true); - } else if (lex_accept(T_plus)) { - int index = read_const_expr(initializer_scope); - - val = compute_element_address(parent, bb, val, - index, stride); - } else if (lex_accept(T_minus)) { - int index = read_const_expr(initializer_scope); - - val = compute_element_address(parent, bb, val, - -index, stride); - } - } else { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - val = opstack_pop(); - } - } - } + val = read_array_scalar_initializer( + parent, bb, !runtime_initializer, initializer_scope); } if (is_implicit && count >= MAX_IMPLICIT_ARRAY) @@ -2610,6 +2456,7 @@ void parse_array_init(var_t *var, block_t *parent, basic_block_t **bb) ? var->func_signature : var->type->func_signature; diagnose_function_pointer_conversion(val, &element); + diagnose_callback_slot_initializer(val, &element); } else if (val && (val->is_func || val->func_signature || (val->type && val->type->func_signature)) && @@ -2667,34 +2514,53 @@ void parse_array_init(var_t *var, block_t *parent, basic_block_t **bb) else v = resize_to(parent, bb, val, var->type, 0); - var_t *elem_addr = compute_element_address( - parent, bb, base_addr, count, elem_size); + bool already_zeroed = + parent != GLOBAL_BLOCK && !is_implicit && !sparse_zero && + count >= initialized_end && !var->is_volatile && + !var->type->is_volatile_qualified && + !var->type->array_element_is_volatile && !var->ptr_level && + !var->is_func && !is_record_type(var->type) && + v->is_const && !v->init_val && !v->init_val_hi; /* A direct long long element is wider than a pointer on a * 32-bit target, where it is written as a register pair. */ - if (elem_size <= PTR_SIZE || - (elem_size == 8 && !var->ptr_level && !var->is_func && - !is_record_type(var->type))) { - add_insn(parent, *bb, OP_write, NULL, elem_addr, v, - elem_size, NULL); - } else { - fatal("Unsupported: array element wider than a pointer"); + if (!already_zeroed) { + var_t *elem_addr = compute_element_address( + parent, bb, base_addr, count, elem_size); + + if (elem_size <= PTR_SIZE || + (elem_size == 8 && !var->ptr_level && !var->is_func && + !is_record_type(var->type))) { + add_insn(parent, *bb, OP_write, NULL, elem_addr, v, + elem_size, NULL); + } else { + fatal( + "Unsupported: array element wider than a pointer"); + } } } + if (var->is_compound_literal && count == 0 && val) + var->init_val = val->init_val; + count++; + if (count > initialized_end) + initialized_end = count; if (is_implicit && count > inferred_size) inferred_size = count; - if (!elided_comma && !lex_accept(T_comma)) - break; - if (lex_peek(T_close_curly, NULL)) + if (!initializer_separator(elided_comma) || + lex_peek(T_close_curly, NULL)) break; } } lex_expect(T_close_curly); + if (runtime_initializer && sparse_zero && !is_implicit) + emit_zero_elements(parent, bb, base_addr, var, initialized_end, + var->array_size, elem_size); + if (is_implicit) { /* `count` walks scalars, so a brace-elided list that stops inside a * row, as in `int m[][2] = { 1, 2, 3 }`, leaves inferred_size short of @@ -2704,8 +2570,8 @@ void parse_array_init(var_t *var, block_t *parent, basic_block_t **bb) int inner = fixed_array_inner_count(var); int whole = (inferred_size + inner - 1) / inner * inner; - if (parent != GLOBAL_BLOCK) - emit_zero_elements(parent, bb, base_addr, inferred_size, whole, + if (runtime_initializer) + emit_zero_elements(parent, bb, base_addr, var, inferred_size, whole, elem_size); inferred_size = whole; var->array_size = inferred_size; @@ -2714,114 +2580,6 @@ void parse_array_init(var_t *var, block_t *parent, basic_block_t **bb) global_constant_initializer_scope = saved_initializer_scope; } -void parse_array_compound_literal(var_t *var, - block_t *parent, - basic_block_t **bb) -{ - int elem_size = var->type->size; - int count = 0; - - reject_empty_initializer_in_strict_c99(); - - /* A compound literal may spell either an inferred bound, ``int[]``, or an - * actual array type, ``int[4]``. The latter is not merely syntax: omitted - * members are zero-initialized and an excess initializer is a constraint - * violation. Keep the parsed bound until the initializer has been consumed; - * previously this routine reset it and silently turned every declared-bound - * literal into an inferred-size one. - */ - int declared_size = var->array_size; - int inferred_size = 0; - var->init_val = 0; - - /* The opening brace is already consumed. See parse_array_init(). */ - if (string_ends_braced_initializer(var, cur_token->next)) { - if (lex_peek(T_wstring, NULL)) - parse_wstring_array_init(var, parent, bb); - else - parse_string_array_init(var, parent, bb); - lex_accept(T_comma); - lex_expect(T_close_curly); - return; - } - - /* A designated element can leave holes before or after it, so initialize - * the declared object before parsing any explicit elements. - */ - emit_zero_elements(parent, bb, var, 0, declared_size, elem_size); - - if (!lex_peek(T_close_curly, NULL)) { - for (;;) { - bool elided_comma = false; - - if (read_array_designator(parent, var, 1, !declared_size, &count, - NULL)) - lex_expect(T_assign); - if (declared_size && count >= declared_size) - error_at("Too many elements in array compound literal", - next_token_loc()); - if (!declared_size && count >= MAX_IMPLICIT_ARRAY) - error_at("Too many elements in array compound literal", - next_token_loc()); - - /* An inferred-bound array gets its size only after the closing - * brace. Still zero every gap before storing a designator so the - * automatic object obeys C99's aggregate initialization rule. - * inferred_size is one past the highest initialized slot, so a - * later backward designator cannot make a forward gap overwrite an - * earlier explicit value. - */ - if (!declared_size) - emit_zero_elements(parent, bb, var, inferred_size, count, - elem_size); - - var_t *elem_addr = - compute_element_address(parent, bb, var, count, elem_size); - if (lex_peek(T_open_curly, NULL) && is_record_type(var->type)) { - /* The array compound literal owns a real aggregate object, just - * like an ordinary array initializer. A braced element must - * therefore be lowered through the shared record path; treating - * it as an expression rejected the opening brace and made - * (struct S[]){ { ... }, { ... } } unusable. - */ - type_t *record_type = resolve_record_type(var->type); - - lex_expect(T_open_curly); - parse_struct_field_init(parent, bb, record_type, elem_addr); - lex_expect(T_close_curly); - } else if (unbraced_record_starts_here(var)) { - elided_comma = parse_unbraced_record_init( - parent, bb, var->type, elem_addr, - declared_size > 0 || count < inferred_size); - } else { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - var_t *value = opstack_pop(); - if (count == 0) - var->init_val = value->init_val; - - var_t *store_val = resize_to(parent, bb, value, var->type, 0); - add_insn(parent, *bb, OP_write, NULL, elem_addr, store_val, - elem_size, NULL); - } - - if (!declared_size) { - if (count + 1 > inferred_size) - inferred_size = count + 1; - } - count++; - if (!elided_comma && !lex_accept(T_comma)) - break; - if (lex_peek(T_close_curly, NULL)) - break; - } - } - - lex_expect(T_close_curly); - - var->array_size = declared_size ? declared_size : inferred_size; -} - /* Identify compiler-emitted temporaries that hold array compound literals. They * keep array metadata without pointer indirection and are marked via * is_compound_literal when synthesized. @@ -2873,9 +2631,7 @@ var_t *scalarize_array_literal(block_t *parent, /* Create a new scalar temporary, giving it a unique name and copying over * the literal data so downstream code can treat it like a normal value. */ - var_t *scalar = require_typed_var(parent, result_type); - scalar->ptr_level = 0; - scalar->var_name = gen_name(); + var_t *scalar = name_var(require_typed_ptr_var(parent, result_type, 0)); scalar->init_val = array_var->init_val; /* Materialize the literal data into the scalar temporary via an OP_read. */ diff --git a/src/parser-sizeof.c b/src/parser-sizeof.c index 0d386d5c..d47ebddb 100644 --- a/src/parser-sizeof.c +++ b/src/parser-sizeof.c @@ -75,13 +75,11 @@ int sizeof_grouped_literal_depth(token_t *token, token_kind_t kind) */ static void push_sizeof_result(block_t *parent, basic_block_t *bb, int size) { - var_t *result = require_typed_var(parent, find_type("size_t", true)); + var_t *result = + load_constant(parent, bb, size, find_builtin_type("size_t")); - result->init_val = size; result->is_const = true; - result->var_name = gen_name(); opstack_push(result); - add_insn(parent, bb, OP_load_constant, result, NULL, NULL, 0, NULL); } int sizeof_array_object(const var_t *array) @@ -93,6 +91,22 @@ int sizeof_array_object(const var_t *array) return array->array_size * element_size; } +static void sizeof_expression_type(var_t *expr, + token_t *sizeof_token, + type_t **type, + int *ptr_cnt, + int *array_size, + bool *is_function) +{ + if (is_bitfield(expr)) + error_at("sizeof cannot be applied to a bit-field", + &sizeof_token->location); + *type = expr->type; + *ptr_cnt = expr->ptr_level; + *array_size = expr->array_size; + *is_function = expr->is_func; +} + /* What a type-only walk over a sizeof operand has seen. */ typedef struct { int operators; /* unary, postfix and cast operators applied */ @@ -198,8 +212,7 @@ static void sizeof_walk_address(var_t *object, sizeof_walk_t *walk, token_t *op) fixed_array_shape_t shape = fixed_array_shape_from_var(object); fixed_array_shape_t scalar = {0}; - fixed_array_shape_to_pointee_var(object, &shape); - object->pointee_array_element_ptr_level = object->ptr_level; + set_pointee_array_shape(object, &shape, object->ptr_level); fixed_array_shape_to_var(object, &scalar); } object->ptr_level++; @@ -231,9 +244,7 @@ static type_t *read_sizeof_cast_type(block_t *scope, int *ptr_level) while (type && lex_accept(T_asterisk)) { stars++; - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; + skip_type_qualifiers(); } *ptr_level = stars; if (!type || type->is_floating || type->array_size || @@ -357,8 +368,7 @@ static bool scan_sizeof_postfix_operand(block_t *scope, fixed_array_shape_drop_outer(&shape); fixed_array_shape_to_var(object, &scalar); if (shape.rank) { - fixed_array_shape_to_pointee_var(object, &shape); - object->pointee_array_element_ptr_level = root->ptr_level; + set_pointee_array_shape(object, &shape, root->ptr_level); } object->ptr_level++; walk->operators++; @@ -672,7 +682,7 @@ void handle_sizeof_operator(block_t *parent, basic_block_t **bb) !strcmp(token, "__func__")) { lex_expect(T_identifier); push_sizeof_result(parent, *bb, - strlen(parent->func->return_def.var_name) + 1); + strlen(function_source_name(parent->func)) + 1); return; } @@ -691,13 +701,8 @@ void handle_sizeof_operator(block_t *parent, basic_block_t **bb) read_expr_operand(parent, &unevaluated_bb); unevaluated_expression_depth--; var_t *expr_var = opstack_pop(); - if (is_bitfield(expr_var)) - error_at("sizeof cannot be applied to a bit-field", - &sizeof_tk->location); - type = expr_var->type; - ptr_cnt = expr_var->ptr_level; - array_size = expr_var->array_size; - is_function = expr_var->is_func; + sizeof_expression_type(expr_var, sizeof_tk, &type, &ptr_cnt, + &array_size, &is_function); if (type == TY_void && ptr_cnt == 0) error_at("sizeof(void) is invalid", &sizeof_tk->location); if (is_function && ptr_cnt == 0) @@ -725,7 +730,7 @@ void handle_sizeof_operator(block_t *parent, basic_block_t **bb) lex_expect(T_identifier); lex_expect(T_close_bracket); push_sizeof_result(parent, *bb, - strlen(parent->func->return_def.var_name) + 1); + strlen(function_source_name(parent->func)) + 1); return; } @@ -773,19 +778,11 @@ void handle_sizeof_operator(block_t *parent, basic_block_t **bb) /* sizeof(expression) - parse the expression and get its type */ basic_block_t *unevaluated_bb = bb_create(parent); unevaluated_expression_depth++; - if (!read_assignment_expression(parent, &unevaluated_bb)) { - read_expr(parent, &unevaluated_bb); - read_ternary_operation(parent, &unevaluated_bb); - } + read_assignment_or_expression(parent, &unevaluated_bb); unevaluated_expression_depth--; var_t *expr_var = opstack_pop(); - if (is_bitfield(expr_var)) - error_at("sizeof cannot be applied to a bit-field", - &sizeof_tk->location); - type = expr_var->type; - ptr_cnt = expr_var->ptr_level; - array_size = expr_var->array_size; - is_function = expr_var->is_func; + sizeof_expression_type(expr_var, sizeof_tk, &type, &ptr_cnt, &array_size, + &is_function); if (is_incomplete_record_object(expr_var)) error_at("sizeof cannot be applied to an incomplete record type", &sizeof_tk->location); diff --git a/src/parser-stmt.c b/src/parser-stmt.c index e62da729..e306cdbf 100644 --- a/src/parser-stmt.c +++ b/src/parser-stmt.c @@ -87,8 +87,7 @@ bool constant_expression_needs_typed_value(token_t *token) (bracket_depth == 0 && token->kind == T_comma)) return false; else if (token->kind == T_numeric && - (numeric_literal_needs_wide_path(token->literal) || - numeric_literal_needs_typed_global_path(token->literal))) + numeric_literal_needs_global_reader(token->literal)) return true; } return false; @@ -158,16 +157,14 @@ basic_block_t *read_switch_label_statement(block_t *parent, basic_block_t *body) context->seen_cases = seen; } - constant = - require_typed_var(context->dispatch_parent, context->control->type); - constant->var_name = gen_name(); + constant = name_var(require_typed_var(context->dispatch_parent, + context->control->type)); constant->init_val = case_lo; constant->init_val_hi = case_hi; add_insn(context->dispatch_parent, context->dispatch_tail, OP_load_constant, constant, NULL, NULL, 0, NULL); - comparison = require_var(context->dispatch_parent); - comparison->var_name = gen_name(); + comparison = require_named_var(context->dispatch_parent); add_insn(context->dispatch_parent, context->dispatch_tail, OP_eq, comparison, constant, context->control, 0, NULL); add_insn(context->dispatch_parent, context->dispatch_tail, OP_branch, @@ -341,6 +338,49 @@ typedef struct { bool is_volatile; } block_decl_specifiers_t; +static bool read_block_decl_specifiers(block_decl_specifiers_t *spec) +{ + bool saw_specifier = false; + + while (lex_peek(T_static, NULL) || lex_peek(T_extern, NULL) || + lex_peek(T_const, NULL) || lex_peek(T_volatile, NULL) || + lex_peek(T_register, NULL) || lex_peek(T_auto, NULL)) { + saw_specifier = true; + if (lex_accept(T_static)) { + if (spec->is_static) + error_at("duplicate static storage class specifier", + cur_token_loc()); + spec->is_static = true; + } else if (lex_accept(T_extern)) { + if (spec->is_extern) + error_at("duplicate extern storage class specifier", + cur_token_loc()); + spec->is_extern = true; + } else if (lex_accept(T_register)) { + if (spec->is_register) + error_at("duplicate register storage class specifier", + cur_token_loc()); + spec->is_register = true; + } else if (lex_accept(T_auto)) { + if (spec->is_auto) + error_at("duplicate auto storage class specifier", + cur_token_loc()); + spec->is_auto = true; + } else if (lex_accept(T_volatile)) { + spec->is_volatile = true; + } else { + lex_expect(T_const); + spec->is_const = true; + } + } + if ((spec->is_static && + (spec->is_register || spec->is_extern || spec->is_auto)) || + (spec->is_register && (spec->is_extern || spec->is_auto)) || + (spec->is_extern && spec->is_auto)) + error_at("incompatible storage class specifiers", cur_token_loc()); + return saw_specifier; +} + static basic_block_t *read_block_declarators( block_t *parent, basic_block_t *bb, @@ -354,12 +394,7 @@ basic_block_t *handle_for_statement(block_t *parent, basic_block_t *bb) char token[MAX_ID_LEN]; type_t *type; var_t *vd; - bool is_const = false; - bool is_static = false; - bool is_extern = false; - bool is_register = false; - bool is_auto = false; - bool is_volatile = false; + block_decl_specifiers_t spec = {0}; bool saw_decl_specifier = false; bool for_decl_semicolon_consumed = false; @@ -383,42 +418,8 @@ basic_block_t *handle_for_statement(block_t *parent, basic_block_t *bb) * emits no setup IR. */ handle_block_typedef_statement(blk, setup); - for_decl_semicolon_consumed = true; } else if (!lex_accept(T_semicolon)) { - while (lex_peek(T_static, NULL) || lex_peek(T_extern, NULL) || - lex_peek(T_const, NULL) || lex_peek(T_volatile, NULL) || - lex_peek(T_register, NULL) || lex_peek(T_auto, NULL)) { - saw_decl_specifier = true; - if (lex_accept(T_static)) { - if (is_static) - error_at("duplicate static storage class specifier", - cur_token_loc()); - is_static = true; - } else if (lex_accept(T_extern)) { - if (is_extern) - error_at("duplicate extern storage class specifier", - cur_token_loc()); - is_extern = true; - } else if (lex_accept(T_register)) { - if (is_register) - error_at("duplicate register storage class specifier", - cur_token_loc()); - is_register = true; - } else if (lex_accept(T_auto)) { - if (is_auto) - error_at("duplicate auto storage class specifier", - cur_token_loc()); - is_auto = true; - } else if (lex_accept(T_volatile)) { - is_volatile = true; - } else { - lex_expect(T_const); - is_const = true; - } - } - if ((is_static && (is_register || is_extern || is_auto)) || - (is_register && (is_extern || is_auto)) || (is_extern && is_auto)) - error_at("incompatible storage class specifiers", cur_token_loc()); + saw_decl_specifier = read_block_decl_specifiers(&spec); bool has_builtin_type = lex_peek(T_signed, NULL) || lex_peek(T_unsigned, NULL) || @@ -447,22 +448,14 @@ basic_block_t *handle_for_statement(block_t *parent, basic_block_t *bb) if (!type && saw_decl_specifier) error_at("declaration specifier requires a type", cur_token_loc()); if (type) { - block_decl_specifiers_t spec = {0}; - /* C99 6.8.5.3 admits only automatic or register object declarations * here. The default mode retains its historical block-scope * static/extern extension. */ - if (strict_c99 && (is_static || is_extern)) + if (strict_c99 && (spec.is_static || spec.is_extern)) error_at( "C99 for initializer permits only auto or register objects", cur_token_loc()); - spec.is_const = is_const; - spec.is_static = is_static; - spec.is_extern = is_extern; - spec.is_register = is_register; - spec.is_auto = is_auto; - spec.is_volatile = is_volatile; /* The loop scope takes the ordinary block declaration lowering, * which also consumes the terminating semicolon. @@ -491,20 +484,16 @@ basic_block_t *handle_for_statement(block_t *parent, basic_block_t *bb) lex_expect(T_semicolon); } else { /* always true */ - vd = require_var(blk); - vd->init_val = 1; - vd->var_name = gen_name(); + vd = load_constant(blk, cond_, 1, TY_int); opstack_push(vd); - add_insn(blk, cond_, OP_load_constant, vd, NULL, NULL, 0, NULL); } bb_connect(cond_, for_end, ELSE); - vd = opstack_pop(); - reject_record_operand(vd); - add_insn(blk, cond_, OP_branch, NULL, vd, NULL, 0, NULL); + emit_control_branch(blk, cond_); basic_block_t *inc_ = bb_create(blk); - continue_bb_push(inc_); + basic_block_t *inc_start = inc_; + continue_bb_push(inc_start); /* increment after each loop */ if (!lex_accept(T_close_bracket)) { @@ -520,10 +509,10 @@ basic_block_t *handle_for_statement(block_t *parent, basic_block_t *bb) /* Normal fallthrough from the loop body goes through the increment block. A * continue statement may already have connected another predecessor to - * inc_. + * inc_start. */ if (body_) - bb_connect(body_, inc_, NEXT); + bb_connect(body_, inc_start, NEXT); /* An empty increment block still needs its back-edge when it is reachable * through normal fallthrough or continue. @@ -534,14 +523,7 @@ basic_block_t *handle_for_statement(block_t *parent, basic_block_t *bb) * break; * } */ - bool has_pred = false; - for (int i = 0; i < inc_->prev_idx; i++) { - if (inc_->prev[i].bb) { - has_pred = true; - break; - } - } - if (has_pred) + if (bb_has_pred(inc_start)) bb_connect(inc_, cond_start, NEXT); /* jump to increment */ @@ -553,8 +535,6 @@ basic_block_t *handle_for_statement(block_t *parent, basic_block_t *bb) /* A do-while loop, whose condition is tested after the body. */ basic_block_t *handle_do_statement(block_t *parent, basic_block_t *bb) { - var_t *vd; - basic_block_t *n = bb_create(parent); bb_connect(bb, n, NEXT); bb = n; @@ -570,13 +550,7 @@ basic_block_t *handle_do_statement(block_t *parent, basic_block_t *bb) bb_connect(do_body, cond_, NEXT); lex_expect(T_while); - lex_expect(T_open_bracket); - read_control_expression(parent, &cond_); - lex_expect(T_close_bracket); - - vd = opstack_pop(); - reject_record_operand(vd); - add_insn(parent, cond_, OP_branch, NULL, vd, NULL, 0, NULL); + read_control_branch(parent, &cond_); lex_expect(T_semicolon); @@ -698,7 +672,7 @@ type_t *read_enum_specifier(block_t *parent, bool *is_definition) int val = 0; type_t *type = NULL; bool has_tag = false; - block_t *scope = parent ? parent : GLOBAL_BLOCK; + block_t *scope = parent ? parent : CURRENT_TU_SCOPE; lex_expect(T_enum); if (lex_peek(T_identifier, token)) { @@ -735,6 +709,10 @@ type_t *read_enum_specifier(block_t *parent, bool *is_definition) } first = false; if (!parent) { + if (find_block_ordinary(CURRENT_TU_SCOPE, token, ORDINARY_ANY, + NULL)) + error_at("identifier redeclared as a different kind of symbol", + cur_token_loc()); add_constant(token, val); continue; } @@ -853,9 +831,7 @@ static void emit_scalar_initializer(block_t *parent, var->type->base_type == TYPE_short) && expr_result->var_name[0] == '.') { /* Extract first element from compound literal array */ - var_t *first_elem = require_var(parent); - first_elem->type = var->type; - first_elem->var_name = gen_name(); + var_t *first_elem = name_var(require_typed_var(parent, var->type)); /* Read first element from array at offset 0 expr_result is the array * itself, so we can read directly from it @@ -887,7 +863,6 @@ static void read_block_declarator_storage(block_t *parent, add_insn(spec->is_static ? GLOBAL_BLOCK : parent, spec->is_static ? GLOBAL_FUNC->bbs : *bb, OP_allocat, var, NULL, NULL, 0, NULL); - add_symbol(*bb, var); if (lex_accept(T_assign)) { validate_string_array_initializer(var); if (spec->is_static) { @@ -930,18 +905,14 @@ static void read_block_declarator_storage(block_t *parent, struct_type->base_struct) struct_type = struct_type->base_struct; - var_t *struct_addr = require_var(parent); - struct_addr->var_name = gen_name(); + var_t *struct_addr = require_named_var(parent); add_insn(parent, *bb, OP_address_of, struct_addr, var, NULL, 0, NULL); lex_expect(T_open_curly); parse_struct_field_init(parent, bb, struct_type, struct_addr); lex_expect(T_close_curly); } else { - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } + read_assignment_or_expression(parent, bb); emit_scalar_initializer(parent, bb, var, opstack_pop()); } @@ -1140,41 +1111,7 @@ basic_block_t *handle_declaration(block_t *parent, basic_block_t *bb) type_t *type; block_decl_specifiers_t spec = {0}; - while (lex_peek(T_static, NULL) || lex_peek(T_extern, NULL) || - lex_peek(T_const, NULL) || lex_peek(T_volatile, NULL) || - lex_peek(T_register, NULL) || lex_peek(T_auto, NULL)) { - if (lex_accept(T_static)) { - if (spec.is_static) - error_at("duplicate static storage class specifier", - cur_token_loc()); - spec.is_static = true; - } else if (lex_accept(T_extern)) { - if (spec.is_extern) - error_at("duplicate extern storage class specifier", - cur_token_loc()); - spec.is_extern = true; - } else if (lex_accept(T_register)) { - if (spec.is_register) - error_at("duplicate register storage class specifier", - cur_token_loc()); - spec.is_register = true; - } else if (lex_accept(T_auto)) { - if (spec.is_auto) - error_at("duplicate auto storage class specifier", - cur_token_loc()); - spec.is_auto = true; - } else if (lex_accept(T_volatile)) { - spec.is_volatile = true; - } else { - lex_expect(T_const); - spec.is_const = true; - } - } - if ((spec.is_static && - (spec.is_register || spec.is_extern || spec.is_auto)) || - (spec.is_register && (spec.is_extern || spec.is_auto)) || - (spec.is_extern && spec.is_auto)) - error_at("incompatible storage class specifiers", cur_token_loc()); + read_block_decl_specifiers(&spec); if (floating_type_starts_here()) error_at("Floating point types are not yet supported", cur_token_loc()); @@ -1298,7 +1235,6 @@ basic_block_t *handle_declaration(block_t *parent, basic_block_t *bb) basic_block_t *n = bb_create(parent); bb_connect(bb, n, NEXT); add_label(token, n); - add_insn(parent, n, OP_label, NULL, NULL, NULL, 0, token); return n; } return read_full_expression_statement(parent, bb); @@ -1343,152 +1279,118 @@ static void peek_specifier_qualifiers(bool *is_const, bool *is_volatile) * selected direct-function, callback-pointer, and fixed callback-array forms; * remaining derived declarations await the shared declarator path. */ -basic_block_t *handle_block_typedef_statement(block_t *parent, - basic_block_t *bb) +static type_t *typedef_alias_from_declarator(const var_t *source, + bool specifier_const, + bool specifier_volatile, + bool file_scope) { - var_t decl = {0}; - bool specifier_const = false; - bool specifier_volatile = false; - - lex_expect(T_typedef); - decl.scope = parent; - peek_specifier_qualifiers(&specifier_const, &specifier_volatile); - parsing_block_typedef_declarator = true; - - /* The shared specifier reader resolves an enum tag but cannot define one, - * so read an enum specifier here, then its trailing qualifiers and the - * declarators. A leading qualifier is still ahead of the enum keyword. + var_t decl; + memcpy(&decl, source, sizeof(decl)); + type_t *base = decl.type; + type_t *alias = add_type(); + func_t *direct_function_signature = decl.func_signature; + func_t *callback_signature = decl.func_signature; + func_t *callback_slot_signature = decl.pointee_func_signature; + bool direct_array = decl.has_direct_array_declarator; + bool direct_pointee_array = decl.has_direct_pointee_array_declarator; + + /* `fn_t (*rows_t)[2]`, or the spelled `int (*(*rows_t)[2])(int)`, points to + * a row of callbacks: the callback typedef is the element. */ - read_type_qualifiers(&decl.is_const_qualified, &decl.is_volatile, false); - if (lex_peek(T_enum, NULL)) { - bool is_definition; - - decl.type = read_enum_specifier(parent, &is_definition); - read_type_qualifiers(&decl.is_const_qualified, &decl.is_volatile, - false); - read_inner_var_decl(&decl, false, false, false); - } else if ((lex_peek(T_struct, NULL) || lex_peek(T_union, NULL)) && - (cur_token->next->next->kind == T_open_curly || - (cur_token->next->next->kind == T_identifier && - cur_token->next->next->next->kind == T_open_curly))) { - /* A record body defines its type, and its tag in this block, before the - * declarators; the shared specifier reader only names a tag. - */ - char tag[MAX_ID_LEN]; - base_type_t kind = accept_record_keyword(); - bool has_tag = lex_peek(T_identifier, tag); - - if (has_tag) - lex_expect(T_identifier); - decl.type = read_record_body(parent, kind, has_tag, tag); - read_type_qualifiers(&decl.is_const_qualified, &decl.is_volatile, - false); - read_inner_var_decl(&decl, false, false, false); - } else - read_full_var_decl(&decl, false, false, false); - parsing_block_typedef_declarator = false; - do { - type_t *base = decl.type; - type_t *alias = add_type(); - func_t *direct_function_signature = decl.func_signature; - func_t *callback_signature = decl.func_signature; - func_t *callback_slot_signature = decl.pointee_func_signature; - bool direct_array = decl.has_direct_array_declarator; - bool direct_pointee_array = decl.has_direct_pointee_array_declarator; - - /* `fn_t (*rows_t)[2]`, or the spelled `int (*(*rows_t)[2])(int)`, - * points to a row of callbacks: the callback typedef is the element. - */ - bool callback_row_alias = - direct_pointee_array && base->func_signature && - !base->is_direct_function_type && !base->pointee_func_signature && - !decl.array_size && !decl.pointee_array_element_ptr_level; - bool inherited_pointee_array = base->pointee_array_size != 0; - bool direct_function_alias = - decl.is_direct_function_declarator && decl.is_func && - decl.func_signature && !decl.array_size && - !decl.pointee_array_size && !base->ptr_level && - !is_record_type(base) && !base->is_floating && - !direct_function_signature->returns_aggregate; - bool callback_pointer_alias = - decl.is_func && decl.func_signature && - decl.parenthesized_function_pointer_level == 1 && !decl.ptr_level && - !decl.array_size && !decl.pointee_array_size && !base->ptr_level && - !is_record_type(base) && !base->is_floating && - !callback_signature->va_args && - !callback_signature->returns_aggregate; - bool callback_pointer_realias = - decl.is_func && decl.func_signature && - !decl.parenthesized_function_pointer_level && !decl.ptr_level && - !decl.array_size && !decl.pointee_array_size && !base->ptr_level && - base->func_signature && !base->is_direct_function_type; - - /* `int (**slot_t)(int)` is a pointer-to-callback object, not a callable - * callback pointer. The shared declarator parser has already normalized - * the extra star into decl.ptr_level and retained the prototype as - * pointee metadata. Keep this first exact typedef form equally narrow: - * scalar/void, no arrays or qualifiers; each further star, as in `int - * (***slot_t)(int)`, is one more object pointer. - */ - bool callback_slot_alias = - callback_slot_signature && - decl.parenthesized_function_pointer_level >= 2 && - decl.ptr_level == decl.parenthesized_function_pointer_level - 1 && - !decl.array_size && !decl.pointee_array_size && !base->ptr_level && - !is_record_type(base) && !base->is_floating && - !decl.is_const_qualified && - !decl.parenthesized_function_pointer_inner_qualified && - !callback_slot_signature->va_args && - !callback_slot_signature->returns_aggregate; - bool callback_slot_realias = - decl.pointee_func_signature && - !decl.parenthesized_function_pointer_level && !decl.ptr_level && - !decl.array_size && !decl.pointee_array_size && - base->ptr_level >= 1 && base->pointee_func_signature && - !decl.is_const_qualified && - !decl.parenthesized_function_pointer_inner_qualified && - !decl.parenthesized_function_pointer_restrict; - bool callback_slot_array_alias = - callback_slot_signature && - decl.parenthesized_function_pointer_level == 2 && - decl.ptr_level == 1 && decl.array_size > 0 && - !decl.has_unsized_array && !decl.pointee_array_size && - !base->ptr_level && !base->array_size && !is_record_type(base) && - !base->is_floating && !decl.is_const_qualified && - (!decl.is_const_pointer || - decl.parenthesized_function_pointer_outer_const) && - (!decl.pointer_const_mask || - (decl.parenthesized_function_pointer_outer_const && - decl.pointer_const_mask == 1U)) && - (!decl.is_volatile || - decl.parenthesized_function_pointer_outer_volatile) && - !decl.parenthesized_function_pointer_inner_qualified && - (!decl.parenthesized_function_pointer_restrict || - decl.parenthesized_function_pointer_outer_restrict) && - !callback_slot_signature->va_args && - !callback_slot_signature->returns_aggregate; - bool callback_slot_array_realias = - !decl.parenthesized_function_pointer_level && !decl.ptr_level && - !decl.pointee_array_size && base->array_size > 0 && - base->array_element_ptr_level == 1 && - base->array_element_pointee_func_signature && - !decl.is_const_pointer && !decl.pointer_const_mask && - !decl.parenthesized_function_pointer_restrict; - bool callback_array_alias = - decl.is_func && decl.func_signature && - decl.parenthesized_function_pointer_level == 1 && - decl.array_size > 0 && !decl.ptr_level && - !decl.pointee_array_size && !base->ptr_level && - !is_record_type(base) && !base->is_floating && - !callback_signature->va_args && - !callback_signature->returns_aggregate; - bool callback_array_realias = - decl.is_func && decl.func_signature && - !decl.parenthesized_function_pointer_level && !direct_array && - !decl.ptr_level && !decl.pointee_array_size && - base->array_size > 0 && base->array_element_ptr_level == 1 && - base->func_signature && !base->is_direct_function_type; - + bool callback_row_alias = + direct_pointee_array && base->func_signature && + !base->is_direct_function_type && !base->pointee_func_signature && + !decl.array_size && + (file_scope || !decl.pointee_array_element_ptr_level); + bool inherited_pointee_array = base->pointee_array_size != 0; + bool direct_function_alias = + decl.is_direct_function_declarator && decl.is_func && + decl.func_signature && !decl.array_size && !decl.pointee_array_size && + !base->ptr_level && + (file_scope || (!is_record_type(base) && !base->is_floating)) && + !direct_function_signature->returns_aggregate; + bool callback_pointer_alias = + decl.is_func && decl.func_signature && + decl.parenthesized_function_pointer_level == 1 && !decl.array_size && + (file_scope || !decl.pointee_array_size) && + (file_scope || !base->ptr_level || + (!base->func_signature && !base->pointee_func_signature)) && + (file_scope || (!is_record_type(base) && !base->is_floating)) && + (file_scope || !callback_signature->va_args) && + (file_scope || !callback_signature->returns_aggregate); + bool callback_pointer_realias = + decl.is_func && decl.func_signature && + !decl.parenthesized_function_pointer_level && !decl.ptr_level && + !decl.array_size && !decl.pointee_array_size && !base->ptr_level && + base->func_signature && !base->is_direct_function_type; + + /* `int (**slot_t)(int)` is a pointer-to-callback object, not a callable + * callback pointer. The shared declarator parser has already normalized the + * extra star into decl.ptr_level and retained the prototype as pointee + * metadata. Keep this first exact typedef form equally narrow: scalar/void, + * no arrays or qualifiers; each further star, as in `int (***slot_t)(int)`, + * is one more object pointer. + */ + bool callback_slot_alias = + callback_slot_signature && + decl.parenthesized_function_pointer_level >= 2 && + (file_scope || + decl.ptr_level == decl.parenthesized_function_pointer_level - 1) && + !decl.array_size && !decl.pointee_array_size && + (file_scope || (!base->ptr_level && !is_record_type(base) && + !base->is_floating && !decl.is_const_qualified && + !decl.parenthesized_function_pointer_inner_qualified && + !callback_slot_signature->va_args && + !callback_slot_signature->returns_aggregate)); + bool callback_slot_realias = + decl.pointee_func_signature && + !decl.parenthesized_function_pointer_level && !decl.ptr_level && + !decl.array_size && !decl.pointee_array_size && base->ptr_level >= 1 && + base->pointee_func_signature && !decl.is_const_qualified && + !decl.parenthesized_function_pointer_inner_qualified && + !decl.parenthesized_function_pointer_restrict; + bool callback_slot_array_alias = + callback_slot_signature && + decl.parenthesized_function_pointer_level == 2 && decl.ptr_level == 1 && + decl.array_size > 0 && !decl.has_unsized_array && + !decl.pointee_array_size && !base->ptr_level && !base->array_size && + !is_record_type(base) && !base->is_floating && + !decl.is_const_qualified && + (!decl.is_const_pointer || + decl.parenthesized_function_pointer_outer_const) && + (!decl.pointer_const_mask || + (decl.parenthesized_function_pointer_outer_const && + decl.pointer_const_mask == 1U)) && + (!decl.is_volatile || + decl.parenthesized_function_pointer_outer_volatile) && + !decl.parenthesized_function_pointer_inner_qualified && + (!decl.parenthesized_function_pointer_restrict || + decl.parenthesized_function_pointer_outer_restrict) && + (file_scope || !callback_slot_signature->va_args) && + (file_scope || !callback_slot_signature->returns_aggregate); + bool callback_slot_array_realias = + !decl.parenthesized_function_pointer_level && !decl.ptr_level && + !decl.pointee_array_size && base->array_size > 0 && + base->array_element_ptr_level == 1 && + base->array_element_pointee_func_signature && !decl.is_const_pointer && + !decl.pointer_const_mask && + !decl.parenthesized_function_pointer_restrict; + bool callback_array_alias = + decl.is_func && decl.func_signature && + decl.parenthesized_function_pointer_level == 1 && decl.array_size > 0 && + (file_scope || !decl.pointee_array_size) && + (file_scope || (!decl.ptr_level && !base->ptr_level && + !is_record_type(base) && !base->is_floating)) && + (file_scope || !callback_signature->va_args) && + (file_scope || !callback_signature->returns_aggregate); + bool callback_array_realias = + decl.is_func && decl.func_signature && + !decl.parenthesized_function_pointer_level && !direct_array && + !decl.ptr_level && !decl.pointee_array_size && base->array_size > 0 && + base->array_element_ptr_level == 1 && base->func_signature && + !base->is_direct_function_type; + + if (!file_scope) { if (decl.parenthesized_function_pointer_level > 1 && !callback_slot_alias && !callback_slot_array_alias) error_at( @@ -1548,166 +1450,286 @@ basic_block_t *handle_block_typedef_statement(block_t *parent, if (lex_peek(T_assign, NULL)) error_at("typedef declaration cannot have an initializer", next_token_loc()); - memcpy(alias, base, sizeof(*alias)); - - /* A record alias takes the typedef descriptor, which reaches the layout - * through base_struct, and so does a function alias, whose void or - * scalar base is only its return type. A data alias keeps its base's - * type, as a file-scope one does: with no base_struct to follow, - * dereferencing a pointer alias would otherwise yield the pointer type - * itself. + } + memcpy(alias, base, sizeof(*alias)); + + /* A record alias takes the typedef descriptor, which reaches the layout + * through base_struct, and so does a function alias, whose void or scalar + * base is only its return type. A data alias keeps its base's type, as a + * file-scope one does: with no base_struct to follow, dereferencing a + * pointer alias would otherwise yield the pointer type itself. + */ + bool is_function_alias = + decl.is_func || decl.func_signature || decl.pointee_func_signature || + base->func_signature || base->pointee_func_signature || + base->array_element_pointee_func_signature; + + if (is_function_alias && !file_scope) + alias->base_type = TYPE_typedef; + if (is_record_type(base) && !base->ptr_level) { + alias->base_type = TYPE_typedef; + alias->base_struct = base; + + /* A pointer to the record is no record itself: like `typedef struct S + * *SP` at file scope it finds the members on its base. */ - bool is_function_alias = decl.is_func || decl.func_signature || - decl.pointee_func_signature || - base->func_signature || - base->pointee_func_signature || - base->array_element_pointee_func_signature; - - if (is_function_alias) - alias->base_type = TYPE_typedef; - if (is_record_type(base) && !base->ptr_level) { - alias->base_type = TYPE_typedef; - alias->base_struct = base; - - /* A pointer to the record is no record itself: like `typedef struct - * S *SP` at file scope it finds the members on its base. - */ - if (decl.ptr_level) { - alias->fields = NULL; - alias->num_fields = 0; - } + if (decl.ptr_level) { + alias->fields = NULL; + alias->num_fields = 0; } - alias->ptr_level = base->ptr_level + decl.ptr_level; - alias->pointer_const_mask = - base->pointer_const_mask | - (decl.pointer_const_mask << base->ptr_level); + } + alias->ptr_level = base->ptr_level + decl.ptr_level; + if (file_scope) + alias->is_direct_function_type = false; + alias->pointer_const_mask = + base->pointer_const_mask | (decl.pointer_const_mask << base->ptr_level); + alias->pointer_volatile_mask = decl.pointer_volatile_mask; + if (file_scope && decl.ptr_level && base->func_signature && + !base->is_direct_function_type && !base->ptr_level && + decl.callback_is_volatile) + alias->pointer_volatile_mask |= 1U; + + /* `const ptr_t` qualifies the pointer that the base typedef hides, not its + * pointee, for every declarator of the list. + */ + if (specifier_const && base->ptr_level && base->ptr_level <= 32 && + !is_function_alias) + alias->pointer_const_mask |= 1U << (base->ptr_level - 1); + alias->is_const_qualified = decl.is_const_qualified; + if (callback_pointer_realias && decl.is_const_qualified) { + /* A callback alias itself is a pointer type, even though its compact + * descriptor stores no ordinary pointer depth. + */ + alias->pointer_const_mask |= 1U; + alias->is_const_qualified = false; + } + alias->is_volatile_qualified = + decl.is_volatile || base->is_volatile_qualified; + int volatile_depth = + base->ptr_level + (base->func_signature && + !base->is_direct_function_type && !base->ptr_level); + if (specifier_volatile && volatile_depth && volatile_depth <= 32) { + alias->pointer_volatile_mask |= 1U << (volatile_depth - 1); + alias->is_volatile_qualified = base->is_volatile_qualified; + } + if (direct_function_alias) { + alias->base_type = TYPE_typedef; - /* `const ptr_t` qualifies the pointer that the base typedef hides, not - * its pointee, for every declarator of the list. + /* The stars of `char *name_t(void)` belong to the return type, which + * the signature already records. */ - if (specifier_const && base->ptr_level && base->ptr_level <= 32 && - !is_function_alias) - alias->pointer_const_mask |= 1U << (base->ptr_level - 1); - alias->is_const_qualified = decl.is_const_qualified; - if (callback_pointer_realias && decl.is_const_qualified) { - /* A callback alias itself is a pointer type, even though its - * compact descriptor stores no ordinary pointer depth. - */ - alias->pointer_const_mask |= 1U; - alias->is_const_qualified = false; - } - alias->is_volatile_qualified = - decl.is_volatile || base->is_volatile_qualified; - if (direct_function_alias) { - /* The stars of `char *name_t(void)` belong to the return type, - * which the signature already records. - */ - alias->ptr_level = base->ptr_level; - alias->pointer_const_mask = base->pointer_const_mask; - alias->func_signature = decl.func_signature; - alias->is_direct_function_type = true; - } - if (callback_pointer_alias) { - alias->size = PTR_SIZE; - alias->func_signature = decl.func_signature; - alias->is_direct_function_type = false; - } - if (callback_slot_alias) { - alias->size = PTR_SIZE; - alias->alignment = PTR_SIZE; - alias->func_signature = NULL; - alias->pointee_func_signature = callback_slot_signature; - alias->is_direct_function_type = false; + alias->ptr_level = base->ptr_level; + alias->pointer_const_mask = base->pointer_const_mask; + if (file_scope) + alias->pointer_volatile_mask = 0; + alias->func_signature = decl.func_signature; + alias->is_direct_function_type = true; + } + if (callback_pointer_alias || (file_scope && callback_array_alias)) { + int return_depth = effective_pointer_depth(&decl); + alias->ptr_level = 0; + alias->pointer_const_mask = + decl.ptr_level < 32 ? decl.pointer_const_mask >> decl.ptr_level : 0; + alias->pointer_volatile_mask = + return_depth < 32 ? decl.pointer_volatile_mask >> return_depth : 0; + if (base->func_signature && !base->is_direct_function_type && + !base->ptr_level) { + alias->pointer_const_mask = decl.callback_is_const ? 1U : 0; + alias->pointer_volatile_mask = decl.callback_is_volatile ? 1U : 0; } - if (callback_slot_array_alias) { - alias->ptr_level = 0; - alias->size = PTR_SIZE; - alias->alignment = PTR_SIZE; - alias->func_signature = NULL; - alias->pointee_func_signature = NULL; - alias->is_direct_function_type = false; - alias->pointer_const_mask = 0; - alias->is_volatile_qualified = false; + alias->is_const_qualified = false; + alias->is_volatile_qualified = false; + alias->size = PTR_SIZE; + alias->alignment = PTR_SIZE; + alias->func_signature = decl.func_signature; + alias->pointee_func_signature = NULL; + alias->is_direct_function_type = false; + } + if (file_scope && (callback_pointer_alias || callback_array_alias || + callback_slot_alias)) { + fixed_array_shape_t shape = fixed_array_shape_from_var(&decl); + fixed_array_shape_to_type(alias, &shape); + alias->array_element_ptr_level = decl.array_size ? 1 : 0; + alias->array_element_type = NULL; + alias->array_element_pointee_func_signature = NULL; + alias->array_element_is_const_pointer = false; + alias->array_element_is_volatile = false; + var_t callback_shape = {0}; + copy_pointee_array_shape_to_type(alias, &callback_shape); + alias->pointee_array_element_type = NULL; + } + if (callback_slot_alias) { + alias->size = PTR_SIZE; + alias->alignment = PTR_SIZE; + alias->func_signature = NULL; + alias->pointee_func_signature = callback_slot_signature; + alias->is_direct_function_type = false; + } + if (callback_slot_array_alias) { + alias->ptr_level = 0; + alias->size = PTR_SIZE; + alias->alignment = PTR_SIZE; + alias->func_signature = NULL; + alias->pointee_func_signature = NULL; + alias->is_direct_function_type = false; + alias->pointer_const_mask = 0; + alias->is_volatile_qualified = false; + compose_block_typedef_array(alias, base, &decl); + alias->array_element_ptr_level = 1; + alias->array_element_pointee_func_signature = callback_slot_signature; + alias->array_element_is_const_pointer = + decl.parenthesized_function_pointer_outer_const; + alias->array_element_is_volatile = + decl.parenthesized_function_pointer_outer_volatile; + } + if (callback_slot_array_realias && direct_array) { + /* Wrapping a completed slot-array alias prepends bounds but must keep + * its element as a callback slot. compose_* records the declaration's + * zero direct pointer depth, so restore the completed element + * descriptor after composition. + */ + compose_block_typedef_array(alias, base, &decl); + alias->array_element_ptr_level = base->array_element_ptr_level; + alias->array_element_pointee_func_signature = + base->array_element_pointee_func_signature; + } + if (callback_slot_array_realias) { + /* Qualifying an array typedef qualifies its element type. Keep that + * fact on the final callback-slot lvalue rather than making the array + * object itself const or volatile. `restrict` has no runtime + * representation, but was validated while parsing. + */ + alias->is_const_qualified = false; + alias->is_volatile_qualified = false; + alias->array_element_is_const_pointer |= decl.is_const_qualified; + alias->array_element_is_volatile |= decl.is_volatile; + } + if (callback_slot_realias && decl.is_const_pointer) { + /* `slot_t const` qualifies the typedef-hidden outer slot pointer, so do + * not shift the declaration's compact bit past it. + */ + alias->pointer_const_mask |= 1U; + alias->is_const_qualified = false; + } + if (callback_array_alias) { + alias->size = PTR_SIZE; + alias->alignment = PTR_SIZE; + alias->func_signature = decl.func_signature; + alias->is_direct_function_type = false; + } + if (file_scope && direct_array && !alias->ptr_level && !alias->size && + alias->base_struct && !alias->base_struct->size) + error_at("Typedef array element has incomplete record type", + cur_token_loc()); + if (direct_array || callback_array_alias) { + if (!file_scope || !callback_array_alias) compose_block_typedef_array(alias, base, &decl); + if (callback_array_alias) { alias->array_element_ptr_level = 1; - alias->array_element_pointee_func_signature = - callback_slot_signature; - alias->array_element_is_const_pointer = - decl.parenthesized_function_pointer_outer_const; - alias->array_element_is_volatile = - decl.parenthesized_function_pointer_outer_volatile; + if (file_scope) + alias->array_element_type = NULL; + } else if (file_scope) { + alias->array_element_ptr_level = alias->ptr_level; + alias->array_element_type = + base->array_element_type + ? base->array_element_type + : (base->ptr_level ? pointee_type_from_pointer_typedef(base) + : base); } - if (callback_slot_array_realias && direct_array) { - /* Wrapping a completed slot-array alias prepends bounds but must - * keep its element as a callback slot. compose_* records the - * declaration's zero direct pointer depth, so restore the completed - * element descriptor after composition. - */ - compose_block_typedef_array(alias, base, &decl); - alias->array_element_ptr_level = base->array_element_ptr_level; - alias->array_element_pointee_func_signature = - base->array_element_pointee_func_signature; - } - if (callback_slot_array_realias) { - /* Qualifying an array typedef qualifies its element type. Keep that - * fact on the final callback-slot lvalue rather than making the - * array object itself const or volatile. `restrict` has no runtime - * representation, but was validated while parsing. - */ - alias->is_const_qualified = false; - alias->is_volatile_qualified = false; - alias->array_element_is_const_pointer |= decl.is_const_qualified; - alias->array_element_is_volatile |= decl.is_volatile; + if (direct_array && decl.ptr_level && decl.pointee_array_size && + base->array_size) { + copy_pointee_array_shape_to_type(alias, &decl); + alias->pointee_array_element_ptr_level = + base->array_element_ptr_level; + alias->pointee_array_element_type = + base->array_element_type ? base->array_element_type : base; + alias->array_element_type = alias->pointee_array_element_type; + if (alias->array_element_type->func_signature) + alias_callback_row_pointer(alias, alias->array_element_type, + decl.pointer_const_mask); } - if (callback_slot_realias && decl.is_const_pointer) { - /* `slot_t const` qualifies the typedef-hidden outer slot pointer, - * so do not shift the declaration's compact bit past it. + if (direct_array && inherited_pointee_array) { + /* A direct array suffix around `int (*row_t)[N]` stores + * pointer-to-row elements. Keep the row descriptor on the completed + * array and mark the selected element as that pointer slot so its + * later load retains the row stride. */ - alias->pointer_const_mask |= 1U; - alias->is_const_qualified = false; - } - if (callback_array_alias) { - alias->size = PTR_SIZE; - alias->alignment = PTR_SIZE; - alias->func_signature = decl.func_signature; - alias->is_direct_function_type = false; + alias->array_element_ptr_level = base->ptr_level; + alias->array_element_type = base->pointee_array_element_type + ? base->pointee_array_element_type + : base; } - if (direct_array || callback_array_alias) { - compose_block_typedef_array(alias, base, &decl); - if (callback_array_alias) - alias->array_element_ptr_level = 1; - if (direct_array && inherited_pointee_array) { - /* A direct array suffix around `int (*row_t)[N]` stores - * pointer-to-row elements. Keep the row descriptor on the - * completed array and mark the selected element as that pointer - * slot so its later load retains the row stride. - */ - alias->array_element_ptr_level = base->ptr_level; - alias->array_element_type = - base->pointee_array_element_type - ? base->pointee_array_element_type - : base; - } - } - if (callback_row_alias) - alias_callback_row_pointer(alias, base, decl.pointer_const_mask); + } + if (callback_row_alias) + alias_callback_row_pointer(alias, base, decl.pointer_const_mask); + + /* As at file scope, `typedef arr_t *rows_t` points to a row of the + * callbacks in the array typedef arr_t. + */ + if (decl.ptr_level == 1 && !direct_array && !direct_pointee_array) + alias_callback_array_pointer(alias, base, decl.pointer_const_mask); + if (direct_pointee_array) { + if (file_scope) + decl.pointee_array_element_ptr_level += base->ptr_level; + copy_pointee_array_shape_to_type(alias, &decl); + alias->pointee_array_element_type = + file_scope && base->ptr_level + ? pointee_type_from_pointer_typedef(base) + : base; + } + if (alias->ptr_level) + alias->size = PTR_SIZE; + return alias; +} + +basic_block_t *handle_block_typedef_statement(block_t *parent, + basic_block_t *bb) +{ + var_t decl = {0}; + bool specifier_const = false; + bool specifier_volatile = false; + + lex_expect(T_typedef); + decl.scope = parent; + peek_specifier_qualifiers(&specifier_const, &specifier_volatile); + parsing_block_typedef_declarator = true; - /* As at file scope, `typedef arr_t *rows_t` points to a row of the - * callbacks in the array typedef arr_t. + /* The shared specifier reader resolves an enum tag but cannot define one, + * so read an enum specifier here, then its trailing qualifiers and the + * declarators. A leading qualifier is still ahead of the enum keyword. + */ + read_type_qualifiers(&decl.is_const_qualified, &decl.is_volatile, false); + if (lex_peek(T_enum, NULL)) { + bool is_definition; + + decl.type = read_enum_specifier(parent, &is_definition); + read_type_qualifiers(&decl.is_const_qualified, &decl.is_volatile, + false); + read_inner_var_decl(&decl, false, false, false); + } else if ((lex_peek(T_struct, NULL) || lex_peek(T_union, NULL)) && + (cur_token->next->next->kind == T_open_curly || + (cur_token->next->next->kind == T_identifier && + cur_token->next->next->next->kind == T_open_curly))) { + /* A record body defines its type, and its tag in this block, before the + * declarators; the shared specifier reader only names a tag. */ - if (decl.ptr_level == 1 && !direct_array && !direct_pointee_array) - alias_callback_array_pointer(alias, base, decl.pointer_const_mask); - if (direct_pointee_array) { - alias->pointee_array_size = decl.pointee_array_size; - alias->pointee_array_dim2 = decl.pointee_array_dim2; - alias->pointee_array_dim3 = decl.pointee_array_dim3; - alias->pointee_array_dim4 = decl.pointee_array_dim4; - alias->pointee_array_element_ptr_level = - decl.pointee_array_element_ptr_level; - alias->pointee_array_element_type = base; - } - if (alias->ptr_level) - alias->size = PTR_SIZE; + char tag[MAX_ID_LEN]; + base_type_t kind = accept_record_keyword(); + bool has_tag = lex_peek(T_identifier, tag); + + if (has_tag) + lex_expect(T_identifier); + decl.type = read_record_body(parent, kind, has_tag, tag); + read_type_qualifiers(&decl.is_const_qualified, &decl.is_volatile, + false); + read_inner_var_decl(&decl, false, false, false); + } else + read_full_var_decl(&decl, false, false, false); + parsing_block_typedef_declarator = false; + do { + type_t *base = decl.type; + type_t *alias = typedef_alias_from_declarator( + &decl, specifier_const, specifier_volatile, false); alias->type_name[0] = '\0'; add_block_typedef(parent, decl.var_name, alias); if (!lex_accept(T_comma)) @@ -1791,7 +1813,7 @@ basic_block_t *read_body_statement(block_t *parent, basic_block_t *bb) return handle_do_statement(parent, bb); if (lex_accept(T_goto)) - return handle_goto_statement(parent, bb); + return handle_goto_statement(bb); hoist_storage_class_specifiers(); if (lex_peek(T_typedef, NULL)) @@ -1801,9 +1823,6 @@ basic_block_t *read_body_statement(block_t *parent, basic_block_t *bb) if (lex_accept(T_semicolon)) return bb; - if (grouped_scalar_pointee_row_store_starts(parent)) - return handle_grouped_scalar_pointee_row_store(parent, bb); - /* These cannot begin a declaration, so they are unambiguously expression * statements. Identifiers remain delegated to handle_declaration(), which * first checks whether they name a type. @@ -1847,8 +1866,6 @@ basic_block_t *read_code_block(func_t *func, block_t *parent, basic_block_t *bb) return bb; } -void var_add_killed_bb(var_t *var, basic_block_t *bb); - void read_func_body(func_t *func) { block_t *blk = add_block(NULL, func); @@ -1859,9 +1876,6 @@ void read_func_body(func_t *func) func->sret_def.type = func->return_def.type; func->sret_def.ptr_level = 1; func->sret_def.var_name = "__shecc_sret"; - func->sret_def.base = &func->sret_def; - add_symbol(func->bbs, &func->sret_def); - var_add_killed_bb(&func->sret_def, func->bbs); } for (int i = 0; i < func->num_params; i++) { @@ -1872,18 +1886,15 @@ void read_func_body(func_t *func) /* A volatile parameter lives in its slot, as a volatile local does. */ if (func->param_defs[i].is_volatile) func->param_defs[i].address_taken = true; - add_symbol(func->bbs, &func->param_defs[i]); - func->param_defs[i].base = &func->param_defs[i]; - var_add_killed_bb(&func->param_defs[i], func->bbs); } + vir_frontend_begin(func); basic_block_t *body = read_code_block(func, NULL, func->bbs); if (body) bb_connect(body, func->exit, NEXT); for (int i = 0; i < backpatch_bb_idx; i++) { basic_block_t *bb = backpatch_bb[i]; - insn_t *g = bb->insn_list.tail; - label_t *label = find_label(g->str); + label_t *label = find_label(backpatch_label[i]); if (!label) error_at("goto label undefined", cur_token_loc()); @@ -1901,4 +1912,5 @@ void read_func_body(func_t *func) backpatch_bb_idx = 0; label_idx = 0; + vir_frontend_end(func); } diff --git a/src/parser.c b/src/parser.c index 5fbb8329..efe41e50 100644 --- a/src/parser.c +++ b/src/parser.c @@ -13,6 +13,8 @@ #include "defs.h" #include "globals.c" +func_t *find_visible_func(char *name, block_t *scope); + /* C language syntactic analyzer */ int global_var_idx = 0; @@ -26,6 +28,7 @@ int continue_pos_idx = 0; label_t labels[MAX_LABELS]; int label_idx = 0; basic_block_t *backpatch_bb[MAX_LABELS]; +char *backpatch_label[MAX_LABELS]; int backpatch_bb_idx = 0; /* stack of the operands of 3AC */ @@ -60,111 +63,165 @@ typedef struct fixed_array_shape { source_location_t *cur_token_loc(void); source_location_t *next_token_loc(void); +/* Advance one initializer scan token, returning false at its boundary. */ +static bool initializer_scan_continue(token_t *token, + int *depth, + bool close_curly) +{ + if (token->kind == T_open_bracket) + (*depth)++; + else if (token->kind == T_close_bracket) { + if (!*depth) + return false; + (*depth)--; + } else if (!*depth && + (token->kind == T_semicolon || token->kind == T_comma || + (close_curly && token->kind == T_close_curly))) + return false; + return true; +} + +static token_t *skip_balanced_delimiter(token_t *token, + token_kind_t open, + token_kind_t close) +{ + int depth = 0; + + for (; token; token = token->next) { + if (token->kind == open) + depth++; + else if (token->kind == close && !--depth) + return token; + } + return NULL; +} + basic_block_t *read_body_statement(block_t *parent, basic_block_t *bb); -static fixed_array_shape_t fixed_array_shape_from_type(const type_t *type) + +/* Sizes and element counts are ints throughout shecc, as sizeof's result is, so + * an array whose count or extent exceeds INT_MAX would wrap: `int + * a[1073741825]` got four bytes and `int a[1073741824]` none. The limit also + * keeps every object below the address space of the 32-bit targets. sizeof + * applies the same bound to the array type names it reads. + */ +static void reject_oversized_array(int count, int scale) +{ + if (scale > 0 && count > INT_MAX / scale) + error_at("Array is too large", cur_token_loc()); +} + +static fixed_array_shape_t fixed_array_shape_from_bounds(int size, + int dim2, + int dim3, + int dim4) { fixed_array_shape_t shape = {0}; int trailing = 1; - if (!type->array_size) + if (!size) return shape; - if (type->array_dim2) - trailing *= type->array_dim2; - if (type->array_dim3) - trailing *= type->array_dim3; - if (type->array_dim4) - trailing *= type->array_dim4; - shape.bounds[shape.rank++] = type->array_size / trailing; - if (type->array_dim2) - shape.bounds[shape.rank++] = type->array_dim2; - if (type->array_dim3) - shape.bounds[shape.rank++] = type->array_dim3; - if (type->array_dim4) - shape.bounds[shape.rank++] = type->array_dim4; + if (dim2) + trailing *= dim2; + if (dim3) + trailing *= dim3; + if (dim4) + trailing *= dim4; + shape.bounds[shape.rank++] = size / trailing; + if (dim2) + shape.bounds[shape.rank++] = dim2; + if (dim3) + shape.bounds[shape.rank++] = dim3; + if (dim4) + shape.bounds[shape.rank++] = dim4; return shape; } -static fixed_array_shape_t fixed_array_shape_from_var(const var_t *var) +static fixed_array_shape_t fixed_array_shape_from_type(const type_t *type) { - type_t shape = {0}; + return fixed_array_shape_from_bounds(type->array_size, type->array_dim2, + type->array_dim3, type->array_dim4); +} - shape.array_size = var->array_size; - shape.array_dim2 = var->array_dim2; - shape.array_dim3 = var->array_dim3; - shape.array_dim4 = var->array_dim4; - return fixed_array_shape_from_type(&shape); +static fixed_array_shape_t fixed_array_shape_from_var(const var_t *var) +{ + return fixed_array_shape_from_bounds(var->array_size, var->array_dim2, + var->array_dim3, var->array_dim4); } static fixed_array_shape_t fixed_array_shape_from_pointee_type( const type_t *type) { - type_t shape = {0}; - - shape.array_size = type->pointee_array_size; - shape.array_dim2 = type->pointee_array_dim2; - shape.array_dim3 = type->pointee_array_dim3; - shape.array_dim4 = type->pointee_array_dim4; - return fixed_array_shape_from_type(&shape); + return fixed_array_shape_from_bounds( + type->pointee_array_size, type->pointee_array_dim2, + type->pointee_array_dim3, type->pointee_array_dim4); } static fixed_array_shape_t fixed_array_shape_from_pointee_var(const var_t *var) { - type_t shape = {0}; - - shape.array_size = var->pointee_array_size; - shape.array_dim2 = var->pointee_array_dim2; - shape.array_dim3 = var->pointee_array_dim3; - shape.array_dim4 = var->pointee_array_dim4; - return fixed_array_shape_from_type(&shape); + return fixed_array_shape_from_bounds( + var->pointee_array_size, var->pointee_array_dim2, + var->pointee_array_dim3, var->pointee_array_dim4); } -static void fixed_array_shape_to_type(type_t *type, - const fixed_array_shape_t *shape) +static void fixed_array_shape_write(const fixed_array_shape_t *shape, + int *size, + int *dim2, + int *dim3, + int *dim4) { - type->array_size = shape->rank ? shape->bounds[0] : 0; - type->array_dim2 = type->array_dim3 = type->array_dim4 = 0; + *size = shape->rank ? shape->bounds[0] : 0; + *dim2 = *dim3 = *dim4 = 0; for (int i = 1; i < shape->rank; i++) { - type->array_size *= shape->bounds[i]; + reject_oversized_array(*size, shape->bounds[i]); + *size *= shape->bounds[i]; if (i == 1) - type->array_dim2 = shape->bounds[i]; + *dim2 = shape->bounds[i]; else if (i == 2) - type->array_dim3 = shape->bounds[i]; + *dim3 = shape->bounds[i]; else - type->array_dim4 = shape->bounds[i]; + *dim4 = shape->bounds[i]; } } +static void fixed_array_shape_to_type(type_t *type, + const fixed_array_shape_t *shape) +{ + fixed_array_shape_write(shape, &type->array_size, &type->array_dim2, + &type->array_dim3, &type->array_dim4); +} + static void fixed_array_shape_to_var(var_t *var, const fixed_array_shape_t *shape) { - var->array_size = shape->rank ? shape->bounds[0] : 0; - var->array_dim2 = var->array_dim3 = var->array_dim4 = 0; - for (int i = 1; i < shape->rank; i++) { - var->array_size *= shape->bounds[i]; - if (i == 1) - var->array_dim2 = shape->bounds[i]; - else if (i == 2) - var->array_dim3 = shape->bounds[i]; - else - var->array_dim4 = shape->bounds[i]; - } + fixed_array_shape_write(shape, &var->array_size, &var->array_dim2, + &var->array_dim3, &var->array_dim4); } static void fixed_array_shape_to_pointee_var(var_t *var, const fixed_array_shape_t *shape) { - var->pointee_array_size = shape->rank ? shape->bounds[0] : 0; - var->pointee_array_dim2 = var->pointee_array_dim3 = - var->pointee_array_dim4 = 0; - for (int i = 1; i < shape->rank; i++) { - var->pointee_array_size *= shape->bounds[i]; - if (i == 1) - var->pointee_array_dim2 = shape->bounds[i]; - else if (i == 2) - var->pointee_array_dim3 = shape->bounds[i]; - else - var->pointee_array_dim4 = shape->bounds[i]; - } + fixed_array_shape_write(shape, &var->pointee_array_size, + &var->pointee_array_dim2, &var->pointee_array_dim3, + &var->pointee_array_dim4); +} + +static void set_pointee_array_shape(var_t *var, + const fixed_array_shape_t *shape, + int element_ptr_level) +{ + fixed_array_shape_to_pointee_var(var, shape); + var->pointee_array_element_ptr_level = element_ptr_level; +} + +static void copy_pointee_array_shape_to_type(type_t *type, const var_t *source) +{ + fixed_array_shape_t shape = fixed_array_shape_from_pointee_var(source); + + fixed_array_shape_write( + &shape, &type->pointee_array_size, &type->pointee_array_dim2, + &type->pointee_array_dim3, &type->pointee_array_dim4); + type->pointee_array_element_ptr_level = + source->pointee_array_element_ptr_level; } static fixed_array_shape_t fixed_array_shape_prepend( @@ -188,21 +245,18 @@ static int fixed_array_shape_stride(const fixed_array_shape_t *shape, int element_size); static bool fixed_array_shape_drop_outer(fixed_array_shape_t *shape); +static int fixed_array_trailing_count(int dim2, int dim3, int dim4) +{ + return (dim2 ? dim2 : 1) * (dim3 ? dim3 : 1) * (dim4 ? dim4 : 1); +} + /* Elements in one outer element of @var's direct array. Only the inner bounds - * matter, which is what keeps this right while an unsized outer bound is still - * being inferred from its initializer and array_size is not yet known. + * matter while an unsized outer bound is inferred from its initializer. */ static int fixed_array_inner_count(const var_t *var) { - int count = 1; - - if (var->array_dim2) - count *= var->array_dim2; - if (var->array_dim3) - count *= var->array_dim3; - if (var->array_dim4) - count *= var->array_dim4; - return count; + return fixed_array_trailing_count(var->array_dim2, var->array_dim3, + var->array_dim4); } /* Rewrite @var's direct array bounds to those of one element of the array: a @@ -220,7 +274,9 @@ static void compose_block_typedef_array(type_t *alias, type_t *base, var_t *decl) { - fixed_array_shape_t base_shape = fixed_array_shape_from_type(base); + fixed_array_shape_t base_shape = {0}; + if (!decl->ptr_level || !decl->pointee_array_size) + base_shape = fixed_array_shape_from_type(base); fixed_array_shape_t decl_shape = fixed_array_shape_from_var(decl); fixed_array_shape_t shape; @@ -274,6 +330,19 @@ static void alias_callback_array_pointer(type_t *alias, basic_block_t *handle_block_typedef_statement(block_t *parent, basic_block_t *bb); +static var_t *scale_pointer_index(block_t *parent, + basic_block_t *bb, + var_t *index, + int stride, + bool constant_scale); +static var_t *lower_array_element_address(block_t *parent, + basic_block_t *bb, + var_t *base, + var_t *index, + type_t *element_type, + int pointer_level, + int stride, + bool rematerialize_scale); bool read_assignment_expression(block_t *parent, basic_block_t **bb); bool is_null_pointer_constant(var_t *value); bool is_record_type(const type_t *type); @@ -333,7 +402,8 @@ void push_object_at(block_t *parent, var_t *lower_reference_update(var_t *object, opcode_t op, block_t *parent, - basic_block_t **bb); + basic_block_t **bb, + bool need_updated_value); void lower_member_postfix(var_t *address, type_t *record_type, bool arrow_first, @@ -441,12 +511,12 @@ bool global_compound_literal_starts_here(void) if (!lex_peek(T_open_bracket, NULL)) return false; next = cur_token->next->next; - if (!next || - !((next->kind == T_identifier && find_type(next->literal, true)) || - next->kind == T_struct || next->kind == T_union || - next->kind == T_enum || next->kind == T_const || - next->kind == T_volatile || next->kind == T_signed || - next->kind == T_unsigned || next->kind == T_long)) + if (!next || !((next->kind == T_identifier && + find_visible_type(next->literal, CURRENT_TU_SCOPE)) || + next->kind == T_struct || next->kind == T_union || + next->kind == T_enum || next->kind == T_const || + next->kind == T_volatile || next->kind == T_signed || + next->kind == T_unsigned || next->kind == T_long)) return false; /* Only a brace after the type name makes a compound literal; otherwise the @@ -474,7 +544,7 @@ bool grouped_global_function_designator_starts_here(bool allow_address) return token && token->kind == T_open_bracket && token->next && token->next->kind == T_identifier && token->next->next && token->next->next->kind == T_close_bracket && - find_func(token->next->literal); + find_visible_func(token->next->literal, CURRENT_TU_SCOPE); } /* `&*function` and `*&function` are both the original function designator. @@ -490,7 +560,8 @@ bool global_function_designator_tokens(token_t *token, if (!token) return false; - if (token->kind == T_identifier && find_func(token->literal)) { + if (token->kind == T_identifier && + find_visible_func(token->literal, CURRENT_TU_SCOPE)) { *after = token->next; *identifier = token; return true; @@ -623,27 +694,39 @@ var_t *require_var(block_t *blk) } var_t *var = arena_calloc(BLOCK_ARENA, 1, sizeof(var_t)); + stats_var_count++; var_list->elements[var_list->size++] = var; /* var_name is a pointer now; every reader dereferences it unconditionally, * so an unnamed variable points at the empty string rather than NULL. */ var->var_name = ""; - var->consumed = -1; - var->phys_reg = -1; - var->phys_reg_hi = -1; - var->first_use = -1; - var->last_use = -1; - var->use_count = 0; - var->base = var; var->type = TY_int; var->scope = blk; var->space_is_allocated = false; var->has_backing_storage = false; - var->ofs_based_on_stack_top = false; return var; } +/* A generated name never matches an identifier, so a temporary just made by + * require_var() leaves its scope's lookup list again. Over a hundred thousand + * of them would otherwise sit in front of every name lookup. + */ +static var_t *name_var(var_t *var) +{ + var_list_t *locals = &((block_t *) var->scope)->locals; + + if (locals->size && locals->elements[locals->size - 1] == var) + locals->size--; + var->var_name = gen_name(); + return var; +} + +static var_t *require_named_var(block_t *blk) +{ + return name_var(require_var(blk)); +} + var_t *require_typed_var(block_t *blk, type_t *type) { if (!type) @@ -654,6 +737,17 @@ var_t *require_typed_var(block_t *blk, type_t *type) return var; } +static var_t *load_constant(block_t *parent, + basic_block_t *bb, + int value, + type_t *type) +{ + var_t *constant = name_var(require_typed_var(parent, type)); + constant->init_val = value; + add_insn(parent, bb, OP_load_constant, constant, NULL, NULL, 0, NULL); + return constant; +} + /* Function-address operands carry a function name, but are not declarations in * the current scope. Keeping them out of the local lookup list lets find_var() * distinguish a resolved function-pointer variable from a generated function @@ -666,6 +760,17 @@ var_t *require_func_symbol_var(block_t *blk) return var; } +/* A resolved function designator used as a parser expression value. */ +var_t *function_designator_value(block_t *block, func_t *func) +{ + var_t *value = require_func_symbol_var(block); + + value->is_func = true; + value->is_function_designator = true; + value->var_name = func->return_def.var_name; + return value; +} + var_t *require_typed_ptr_var(block_t *blk, type_t *type, int ptr) { var_t *var = require_typed_var(blk, type); @@ -812,7 +917,6 @@ type_t *pointee_type_from_pointer_typedef(type_t *type) * based on the represented type. Records retain nominal identity. */ bool compatible_function_signature(const func_t *left, const func_t *right); -func_t *find_visible_func(char *name, block_t *scope); var_t *materialize_function_designator(block_t *parent, basic_block_t **bb, var_t *value); @@ -863,7 +967,147 @@ static bool is_plain_record_alias(const type_t *type) !type->func_signature && !type->pointee_func_signature; } -bool compatible_decl_type(const type_t *left, const type_t *right) +typedef struct compatible_record_pair { + const type_t *left; + const type_t *right; + const struct compatible_record_pair *next; +} compatible_record_pair_t; + +static bool compatible_decl_type_with_records( + const type_t *left, + const type_t *right, + const compatible_record_pair_t *seen); +static bool compatible_function_signature_with_records( + const func_t *left, + const func_t *right, + const compatible_record_pair_t *seen); +static bool compatible_function_param_decl_with_records( + const var_t *left, + const var_t *right, + const compatible_record_pair_t *seen); + +static bool compatible_optional_decl_type_with_records( + const type_t *left, + const type_t *right, + const compatible_record_pair_t *seen) +{ + return !!left == !!right && + (!left || compatible_decl_type_with_records(left, right, seen)); +} + +static bool compatible_var_function_type_with_records( + const var_t *left, + const var_t *right, + const compatible_record_pair_t *seen) +{ + if (left->is_func != right->is_func || + !!left->pointee_func_signature != !!right->pointee_func_signature) + return false; + if (left->is_func && + (callback_pointer_indirection(left) != + callback_pointer_indirection(right) || + !compatible_function_signature_with_records( + left->func_signature, right->func_signature, seen))) + return false; + return !left->pointee_func_signature || + compatible_function_signature_with_records( + left->pointee_func_signature, right->pointee_func_signature, + seen); +} + +static bool compatible_record_member(const var_t *left, + const var_t *right, + const compatible_record_pair_t *seen) +{ + return !strcmp(left->var_name, right->var_name) && + left->ptr_level == right->ptr_level && + compatible_var_function_type_with_records(left, right, seen) && + (!left->is_func || + (left->callback_is_const == right->callback_is_const && + left->callback_is_volatile == right->callback_is_volatile)) && + left->array_size == right->array_size && + left->has_unsized_array == right->has_unsized_array && + left->array_dim2 == right->array_dim2 && + left->array_dim3 == right->array_dim3 && + left->array_dim4 == right->array_dim4 && + left->pointee_array_size == right->pointee_array_size && + left->has_direct_pointee_array_declarator == + right->has_direct_pointee_array_declarator && + left->pointee_array_dim2 == right->pointee_array_dim2 && + left->pointee_array_dim3 == right->pointee_array_dim3 && + left->pointee_array_dim4 == right->pointee_array_dim4 && + left->pointee_array_element_ptr_level == + right->pointee_array_element_ptr_level && + left->type && right->type && + compatible_optional_decl_type_with_records( + left->type->pointee_array_element_type, + right->type->pointee_array_element_type, seen) && + left->is_flexible_array_member == right->is_flexible_array_member && + left->is_const_qualified == right->is_const_qualified && + left->is_volatile == right->is_volatile && + left->pointer_const_mask == right->pointer_const_mask && + left->pointer_volatile_mask == right->pointer_volatile_mask && + left->is_bitfield == right->is_bitfield && + left->bit_width == right->bit_width && + left->bit_storage_size == right->bit_storage_size && + compatible_decl_type_with_records(left->type, right->type, seen); +} + +static bool compatible_record_type(const type_t *left, + const type_t *right, + const compatible_record_pair_t *seen) +{ + compatible_record_pair_t pair; + + if (left->base_type != right->base_type || + strcmp(left->type_name, right->type_name)) + return false; + for (const compatible_record_pair_t *p = seen; p; p = p->next) + if ((p->left == left && p->right == right) || + (p->left == right && p->right == left)) + return true; + + /* A declaration of an incomplete tagged type is compatible with the + * completed declaration in another translation unit. + */ + if (!left->definition_started || !right->definition_started) + return true; + if (left->num_fields != right->num_fields) + return false; + + pair.left = left; + pair.right = right; + pair.next = seen; + if (left->base_type == TYPE_struct) { + for (int i = 0; i < left->num_fields; i++) + if (!compatible_record_member(&left->fields[i], &right->fields[i], + &pair)) + return false; + return true; + } + + bool matched[MAX_FIELDS] = {false}; + for (int i = 0; i < left->num_fields; i++) { + bool found = false; + + for (int j = 0; j < right->num_fields; j++) { + if (matched[j] || !compatible_record_member( + &left->fields[i], &right->fields[j], &pair)) + continue; + matched[j] = true; + found = true; + break; + } + if (!found) + return false; + } + return true; +} + +static bool compatible_decl_type_with_records( + const type_t *left, + const type_t *right, + const compatible_record_pair_t *seen) { /* A typedef of a struct or union tag denotes the tag's type. Its own * qualifiers were copied into each declarator, which the callers compare. @@ -885,8 +1129,8 @@ bool compatible_decl_type(const type_t *left, const type_t *right) left->pointer_const_mask == right->pointer_const_mask && left->is_const_qualified == right->is_const_qualified && left->is_volatile_qualified == right->is_volatile_qualified && - compatible_function_signature(left->func_signature, - right->func_signature); + compatible_function_signature_with_records( + left->func_signature, right->func_signature, seen); /* So may a callback slot typedef, `int (**slot_t)(int)`. */ if (left && right && left->pointee_func_signature && @@ -895,10 +1139,16 @@ bool compatible_decl_type(const type_t *left, const type_t *right) left->pointer_const_mask == right->pointer_const_mask && left->is_const_qualified == right->is_const_qualified && left->is_volatile_qualified == right->is_volatile_qualified && - compatible_function_signature(left->pointee_func_signature, - right->pointee_func_signature); - if (!left || !right || left->base_type != right->base_type || - left->size != right->size || left->ptr_level != right->ptr_level || + compatible_function_signature_with_records( + left->pointee_func_signature, right->pointee_func_signature, + seen); + if (!left || !right) + return false; + if ((left->base_type == TYPE_struct || left->base_type == TYPE_union) && + (right->base_type == TYPE_struct || right->base_type == TYPE_union)) + return compatible_record_type(left, right, seen); + if (left->base_type != right->base_type || left->size != right->size || + left->ptr_level != right->ptr_level || left->is_unsigned != right->is_unsigned || left->is_signed_char != right->is_signed_char || left->is_bool != right->is_bool) @@ -906,10 +1156,8 @@ bool compatible_decl_type(const type_t *left, const type_t *right) if (!!left->func_signature != !!right->func_signature) return false; if (left->func_signature && - !compatible_function_signature(left->func_signature, - right->func_signature)) - return false; - if (left->base_type == TYPE_struct || left->base_type == TYPE_union) + !compatible_function_signature_with_records( + left->func_signature, right->func_signature, seen)) return false; if (left->base_type == TYPE_typedef && (left->num_fields || right->num_fields)) @@ -917,6 +1165,11 @@ bool compatible_decl_type(const type_t *left, const type_t *right) return true; } +bool compatible_decl_type(const type_t *left, const type_t *right) +{ + return compatible_decl_type_with_records(left, right, NULL); +} + /* Function-pointer declarators carry their pointee function type separately * from the pointer-sized var_t representation. Redeclarations must compare that * syntax-only signature as well: comparing just the outer storage shape accepts @@ -924,30 +1177,29 @@ bool compatible_decl_type(const type_t *left, const type_t *right) * side has no prototype, retain the existing old-style policy. */ bool compatible_function_param_decl(const var_t *left, const var_t *right) +{ + return compatible_function_param_decl_with_records(left, right, NULL); +} + +static bool compatible_function_param_decl_with_records( + const var_t *left, + const var_t *right, + const compatible_record_pair_t *seen) { bool left_points_to_value; bool right_points_to_value; - if (!left || !right || left->is_func != right->is_func) + if (!left || !right || + !compatible_var_function_type_with_records(left, right, seen)) return false; if (left->is_func) { if (!left->func_signature || !right->func_signature) return false; - return callback_pointer_indirection(left) == - callback_pointer_indirection(right) && - compatible_function_signature(left->func_signature, - right->func_signature); - } - - if (left->pointee_func_signature || right->pointee_func_signature) { - if (!left->pointee_func_signature || !right->pointee_func_signature || - !compatible_function_signature(left->pointee_func_signature, - right->pointee_func_signature)) - return false; + return true; } - if (!compatible_decl_type(left->type, right->type) || + if (!compatible_decl_type_with_records(left->type, right->type, seen) || left->ptr_level != right->ptr_level) return false; @@ -990,12 +1242,60 @@ bool parameter_changes_under_default_promotion(const var_t *param) } bool compatible_function_signature(const func_t *left, const func_t *right) +{ + return compatible_function_signature_with_records(left, right, NULL); +} + +static bool compatible_function_return_decl_with_records( + const var_t *left, + const var_t *right, + const compatible_record_pair_t *seen) +{ + if (!left || !right || !left->type || !right->type) + return false; + int depth = left->ptr_level + left->type->ptr_level; + if (depth != right->ptr_level + right->type->ptr_level) + return false; + if (left->ptr_level != right->ptr_level || + !compatible_decl_type_with_records(left->type, right->type, seen)) { + if (left->type->func_signature || right->type->func_signature || + left->type->pointee_func_signature || + right->type->pointee_func_signature || left->type->array_size || + right->type->array_size || left->type->pointee_array_size || + right->type->pointee_array_size || left->pointee_array_size || + right->pointee_array_size || + !compatible_decl_type_with_records( + pointee_type_from_pointer_typedef(left->type), + pointee_type_from_pointer_typedef(right->type), seen)) + return false; + } + + /* Scalar and compact callback returns have only top-level qualifiers. An + * ordinary pointer return also keeps qualifiers on its reached objects. + */ + if (!depth) + return true; + unsigned int inner_mask = depth <= 32 ? (1U << (depth - 1)) - 1 : ~0U; + return (left->is_const_qualified || left->type->is_const_qualified) == + (right->is_const_qualified || right->type->is_const_qualified) && + (left->is_volatile || left->type->is_volatile_qualified) == + (right->is_volatile || right->type->is_volatile_qualified) && + (effective_pointer_const_mask(left) & inner_mask) == + (effective_pointer_const_mask(right) & inner_mask) && + ((left->pointer_volatile_mask | left->type->pointer_volatile_mask) & + inner_mask) == ((right->pointer_volatile_mask | + right->type->pointer_volatile_mask) & + inner_mask); +} + +static bool compatible_function_signature_with_records( + const func_t *left, + const func_t *right, + const compatible_record_pair_t *seen) { if (!left || !right || - !compatible_decl_type(left->return_def.type, right->return_def.type) || - left->return_def.ptr_level != right->return_def.ptr_level || - left->return_def.is_const_qualified != - right->return_def.is_const_qualified) + !compatible_function_return_decl_with_records(&left->return_def, + &right->return_def, seen)) return false; if (!left->has_prototype || !right->has_prototype) @@ -1004,8 +1304,8 @@ bool compatible_function_signature(const func_t *left, const func_t *right) left->va_args != right->va_args) return false; for (int i = 0; i < left->num_params; i++) - if (!compatible_function_param_decl(&left->param_defs[i], - &right->param_defs[i])) + if (!compatible_function_param_decl_with_records( + &left->param_defs[i], &right->param_defs[i], seen)) return false; return true; } @@ -1017,7 +1317,13 @@ bool compatible_function_signature(const func_t *left, const func_t *right) */ func_t *find_visible_func(char *name, block_t *scope) { - func_t *func = find_func(name); + var_t *unit_decl = find_tu_declaration(name); + func_t *func; + + if (unit_decl) + func = find_func(unit_decl->var_name); + else + func = find_func(name); var_t *binding; if (!func || !func->is_block_scope_only_declaration) @@ -1100,14 +1406,29 @@ int alignment_var(var_t *var) return alignment_type(var->type); } +/* Round a record offset or size up to @alignment. Each member fits on its own, + * but their sum may still wrap the int. + */ +static int align_record_offset(int size, int alignment) +{ + if (size < 0 || size > INT_MAX - (alignment - 1)) + error_at("Record is too large", cur_token_loc()); + return ALIGN_UP(size, alignment); +} + int layout_struct_field(int size, var_t *field, int *record_alignment) { int alignment = alignment_var(field); + int field_size; if (alignment > *record_alignment) *record_alignment = alignment; - field->offset = ALIGN_UP(size, alignment); - return field->offset + size_var(field); + + field->offset = align_record_offset(size, alignment); + field_size = size_var(field); + if (field_size > INT_MAX - field->offset) + error_at("Record is too large", cur_token_loc()); + return field->offset + field_size; } /* Stateful packing cursor for the C99 bit-field layout path. Ordinary fields @@ -1131,9 +1452,70 @@ typedef struct switch_case_value { } switch_case_value_t; int read_const_expr(block_t *scope); -int read_global_address_offset(block_t *scope, - block_t *parent, - basic_block_t *bb); + +static void set_array_dimension(int *size, + int *dim2, + int *dim3, + int *dim4, + int dimension, + int bound, + bool infer_missing_size) +{ + if (!dimension) { + *size = bound; + } else { + int *dimensions[] = {dim2, dim3, dim4}; + + if (dimension > 3) + dimension = 3; + *dimensions[dimension - 1] = bound; + if (!*size && infer_missing_size) + *size = bound; + else { + reject_oversized_array(*size, bound); + *size *= bound; + } + } +} + +static void read_array_shape_bounds(block_t *scope, + fixed_array_shape_t *shape, + char *rank_error, + char *missing_bound_error, + char *invalid_bound_error, + bool check_rank_first, + bool opening_consumed) +{ + /* Preserve each declarator's error order and any checks after `['. */ + while (opening_consumed || lex_accept(T_open_square)) { + int bound; + + opening_consumed = false; + if (check_rank_first && shape->rank >= MAX_FIXED_ARRAY_RANK) + error_at(rank_error, cur_token_loc()); + if (missing_bound_error && lex_peek(T_close_square, NULL)) + error_at(missing_bound_error, cur_token_loc()); + bound = read_const_expr(scope); + if (!check_rank_first && shape->rank >= MAX_FIXED_ARRAY_RANK) + error_at(rank_error, cur_token_loc()); + if (bound <= 0) + error_at(invalid_bound_error, cur_token_loc()); + shape->bounds[shape->rank++] = bound; + lex_expect(T_close_square); + } +} + +static void fixed_array_shape_apply(const fixed_array_shape_t *shape, + int *size, + int *dim2, + int *dim3, + int *dim4) +{ + /* Unlike fixed_array_shape_write(), retain any unused trailing dimensions. + */ + for (int i = 0; i < shape->rank; i++) + set_array_dimension(size, dim2, dim3, dim4, i, shape->bounds[i], false); +} /* Enumerator values must be representable as int. */ bool checking_enum_constant = false; @@ -1233,12 +1615,12 @@ int layout_bitfield_field(int size, *record_alignment = alignment; if (!field->bit_width) { size = flush_bitfield_layout(size, bits); - return ALIGN_UP(size, alignment); + return align_record_offset(size, alignment); } if (!bits->used_bits || bits->unit_size != field->bit_storage_size || bits->used_bits + field->bit_width > unit_bits) { size = flush_bitfield_layout(size, bits); - bits->unit_offset = ALIGN_UP(size, alignment); + bits->unit_offset = align_record_offset(size, alignment); bits->unit_size = field->bit_storage_size; } field->offset = bits->unit_offset; @@ -1336,12 +1718,12 @@ void discard_operand(block_t *parent, basic_block_t *bb) bool unread = var && var == unread_volatile_object; unread_volatile_object = NULL; - if (!unread || var->is_func || var->array_size || - (!var->ptr_level && is_record_type(var->type))) + if (!unread || var->is_function_designator || var->array_size || + (!var->is_func && !var->ptr_level && is_record_type(var->type))) return; - var_t *copy = require_typed_ptr_var(parent, var->type, var->ptr_level); - copy->var_name = gen_name(); + var_t *copy = name_var(require_typed_ptr_var( + parent, var->type, var->is_func ? 1 : var->ptr_level)); add_insn(parent, bb, OP_assign, copy, var, NULL, 0, NULL); } @@ -1422,87 +1804,36 @@ int get_size(var_t *var) return var->type->size; } -int get_operator_prio(opcode_t op) +opcode_t get_operator(void) { - /* https://www.cs.uic.edu/~i109/Notes/COperatorPrecedenceTable.pdf */ - switch (op) { - case OP_ternary: - return 3; - case OP_log_or: - return 4; - case OP_log_and: - return 5; - case OP_bit_or: - return 6; - case OP_bit_xor: - return 7; - case OP_bit_and: - return 8; - case OP_eq: - case OP_neq: - return 9; - case OP_lt: - case OP_leq: - case OP_gt: - case OP_geq: - return 10; - case OP_lshift: - case OP_rshift: - return 11; - case OP_add: - case OP_sub: - return 12; - case OP_mul: - case OP_div: - case OP_mod: - return 13; - default: - return 0; - } + opcode_t op = cur_token->next ? operator_for_token(cur_token->next->kind) + : OP_generic; + + if (op == OP_bit_not || op == OP_log_not) + return OP_generic; + if (op != OP_generic && op != OP_ternary) + lex_next(); + return op; } -opcode_t get_operator(void) +void fold_integer_constant_cast(var_t *result, + unsigned int lo, + unsigned int hi); + +static void preserve_integer_constant(var_t *result, const var_t *source) { - opcode_t op = OP_generic; - if (lex_accept(T_plus)) - op = OP_add; - else if (lex_accept(T_minus)) - op = OP_sub; - else if (lex_accept(T_asterisk)) - op = OP_mul; - else if (lex_accept(T_divide)) - op = OP_div; - else if (lex_accept(T_mod)) - op = OP_mod; - else if (lex_accept(T_lshift)) - op = OP_lshift; - else if (lex_accept(T_rshift)) - op = OP_rshift; - else if (lex_accept(T_log_and)) - op = OP_log_and; - else if (lex_accept(T_log_or)) - op = OP_log_or; - else if (lex_accept(T_eq)) - op = OP_eq; - else if (lex_accept(T_noteq)) - op = OP_neq; - else if (lex_accept(T_lt)) - op = OP_lt; - else if (lex_accept(T_le)) - op = OP_leq; - else if (lex_accept(T_gt)) - op = OP_gt; - else if (lex_accept(T_ge)) - op = OP_geq; - else if (lex_accept(T_ampersand)) - op = OP_bit_and; - else if (lex_accept(T_bit_or)) - op = OP_bit_or; - else if (lex_accept(T_bit_xor)) - op = OP_bit_xor; - else if (lex_peek(T_question, NULL)) - op = OP_ternary; - return op; + if (!source->is_const || source->ptr_level || source->type->ptr_level || + source->array_size || source->is_func || result->ptr_level || + result->type->ptr_level || is_record_type(source->type) || + is_record_type(result->type) || result->type == TY_void) + return; + unsigned int lo = (unsigned int) source->init_val; + unsigned int hi = source->type->size > TY_int->size + ? (unsigned int) source->init_val_hi + : source->type->is_unsigned ? 0 + : 0U - (lo >> 31); + result->is_const = true; + fold_integer_constant_cast(result, lo, hi); } var_t *promote_unchecked(block_t *block, @@ -1511,8 +1842,7 @@ var_t *promote_unchecked(block_t *block, type_t *target_type, int target_ptr) { - var_t *rd = require_typed_ptr_var(block, target_type, target_ptr); - rd->var_name = gen_name(); + var_t *rd = name_var(require_typed_ptr_var(block, target_type, target_ptr)); /* Encode both source and target sizes in src1: Lower 16 bits: target size * Upper 16 bits: source size This allows codegen to distinguish between @@ -1524,6 +1854,7 @@ var_t *promote_unchecked(block_t *block, else encoded_size |= target_type->size; add_insn(block, *bb, OP_sign_ext, rd, var, NULL, encoded_size, NULL); + preserve_integer_constant(rd, var); return rd; } @@ -1552,10 +1883,10 @@ var_t *truncate_unchecked(block_t *block, type_t *target_type, int target_ptr) { - var_t *rd = require_typed_ptr_var(block, target_type, target_ptr); - rd->var_name = gen_name(); + var_t *rd = name_var(require_typed_ptr_var(block, target_type, target_ptr)); add_insn(block, *bb, OP_trunc, rd, var, NULL, target_ptr ? PTR_SIZE : target_type->size, NULL); + preserve_integer_constant(rd, var); return rd; } @@ -1567,31 +1898,33 @@ var_t *normalize_bool(block_t *block, basic_block_t **bb, var_t *var) if (is_bool_scalar(var->type, var->ptr_level)) return var; if (var->is_const && !var->ptr_level) { - rd = require_typed_var(block, TY_bool); - rd->var_name = gen_name(); - rd->init_val = var->init_val || var->init_val_hi; + rd = load_constant(block, *bb, var->init_val || var->init_val_hi, + TY_bool); rd->is_const = true; - add_insn(block, *bb, OP_load_constant, rd, NULL, NULL, 0, NULL); return rd; } - zero = require_typed_var(block, TY_int); - zero->var_name = gen_name(); - zero->init_val = 0; + zero = load_constant(block, *bb, 0, TY_int); zero->is_const = true; - add_insn(block, *bb, OP_load_constant, zero, NULL, NULL, 0, NULL); - rd = require_typed_var(block, TY_bool); - rd->var_name = gen_name(); + rd = name_var(require_typed_var(block, TY_bool)); add_insn(block, *bb, OP_neq, rd, var, zero, 0, NULL); return rd; } +var_t *resize_to(block_t *block, + basic_block_t **bb, + var_t *val, + type_t *type, + int ptr_level); + /* The value an OP_write of a scalar object of @type should store. A store * narrows to the object width by itself, which is the conversion C wants for * every integer type except _Bool: 0x100 has to become 1, not its low byte 0. - * Paths that write a plain assignment through an address share this rather than - * each resizing their value. + * It does not widen, so an int stored into a long long is converted first: + * otherwise the high word held whatever the int's register did above it, the + * carry of an overflowed sum, say. Paths that write a plain assignment through + * an address share this rather than each resizing their value. */ var_t *convert_stored_value(block_t *block, basic_block_t **bb, @@ -1599,9 +1932,13 @@ var_t *convert_stored_value(block_t *block, type_t *type, int ptr_level) { - if (!is_bool_scalar(type, ptr_level)) - return value; - return normalize_bool(block, bb, value); + if (is_bool_scalar(type, ptr_level)) + return normalize_bool(block, bb, value); + if (!ptr_level && type && !type->ptr_level && !is_record_type(type) && + !value->ptr_level && !value->array_size && !value->is_func && + get_size(value) < type->size) + return resize_to(block, bb, value, type, ptr_level); + return value; } var_t *resize_var(block_t *block, basic_block_t **bb, var_t *from, var_t *to) @@ -1964,8 +2301,11 @@ bool incompatible_pointee_callback_conversion(const var_t *from, effective_pointer_depth(from) > 0 && from->type && from->type->func_signature && !from->type->is_direct_function_type) from_signature = from->type->func_signature; - if (!to_signature && effective_pointer_depth(to) > 0 && to->type && - to->type->func_signature && !to->type->is_direct_function_type) + if (!to_signature && to->type && + (!to->func_signature || + to->func_signature == to->type->func_signature) && + effective_pointer_depth(to) > 0 && to->type->func_signature && + !to->type->is_direct_function_type) to_signature = to->type->func_signature; if (!(from_signature || to_signature)) return false; @@ -1983,12 +2323,60 @@ void diagnose_callback_slot_initializer(const var_t *from, const var_t *to) cur_token_loc()); } +/* Array subscripts undergo integer promotion before their byte offset is + * scaled. Keep that representation on synthetic scaling values: narrowing a + * `unsigned char` product would otherwise wrap before pointer addition. + */ +type_t *integer_promoted_type(const var_t *var) +{ + return var->type->size < TY_int->size ? TY_int : var->type; +} + /* The rest of the parser, in the order it was written. Each file depends on * what the ones before it define, so the order below is load-bearing and must * not be sorted. */ /* clang-format off */ +static void skip_type_qualifiers(void) +{ + while (lex_accept(T_const) || lex_accept(T_volatile) || + lex_accept(T_restrict)) + ; +} + +static token_t *function_pointer_declarator_suffix(token_t *open, + int *star_count); + +static void read_assignment_or_expression(block_t *parent, + basic_block_t **bb); +static void read_comma_expression(block_t *parent, + basic_block_t **bb, + bool clear_reference); +static var_t *read_subscript_address(block_t *parent, + basic_block_t **bb, + var_t *base, + type_t *element_type, + int pointer_level, + int stride) +{ + var_t *index; + + read_assignment_or_expression(parent, bb); + index = opstack_pop(); + lex_expect(T_close_square); + return lower_array_element_address(parent, *bb, base, index, element_type, + pointer_level, stride, false); +} + #include "parser-init.c" +static void read_assignment_or_expression(block_t *parent, + basic_block_t **bb) +{ + if (!read_assignment_expression(parent, bb)) { + read_expr(parent, bb); + read_ternary_operation(parent, bb); + } +} #include "parser-decl.c" #include "parser-call.c" #include "parser-sizeof.c" diff --git a/src/peephole.c b/src/peephole.c index 27b53b6a..cbb87885 100644 --- a/src/peephole.c +++ b/src/peephole.c @@ -9,245 +9,28 @@ #include "defs.h" #include "globals.c" -/* Determines if an instruction can be fused with a following OP_assign. Fusible - * instructions are those whose results can be directly written to the final - * destination register, eliminating intermediate moves. +/* These instructions can write their result straight to a following move's + * destination, eliminating the intermediate register copy. */ -bool is_fusible_insn(const ph2_ir_t *ph2_ir) +bool is_fusible_insn(const vir_mop_t *ir) { - switch (ph2_ir->op) { - case OP_add: /* Arithmetic operations */ - case OP_sub: - case OP_mul: - case OP_div: - case OP_mod: - case OP_lshift: /* Shift operations */ - case OP_rshift: - case OP_bit_and: /* Bitwise operations */ - case OP_bit_or: - case OP_bit_xor: - case OP_log_and: /* Logical operations */ - case OP_log_or: - case OP_log_not: - case OP_negate: /* Unary operations */ - case OP_load: /* Memory operations */ - case OP_global_load: - case OP_load_data_address: - case OP_load_rodata_address: - return true; - default: - return false; - } -} - -/* Opcodes whose "dest" names a register they write. Only opcodes that certainly - * do are listed, so a live register is never mistaken for dead. - */ -bool op_writes_dest(opcode_t op) -{ - switch (op) { - case OP_cmov: - case OP_load: - case OP_load_constant: - case OP_global_load: - case OP_assign: - case OP_add: - case OP_sub: - case OP_mul: - case OP_div: - case OP_mod: - case OP_lshift: - case OP_rshift: - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: - case OP_bit_not: - case OP_negate: - case OP_log_not: - case OP_eq: - case OP_neq: - case OP_lt: - case OP_leq: - case OP_gt: - case OP_geq: - case OP_read: - case OP_address_of: - case OP_global_address_of: - case OP_trunc: - case OP_sign_ext: - return true; - default: - return false; - } -} - -/* Opcodes whose src0 holds something other than a register number. Anything not - * listed is assumed to read src0, which only costs a missed rewrite. - */ -bool op_src0_is_reg(opcode_t op) -{ - switch (op) { - case OP_load: - case OP_load_constant: - case OP_global_load: - case OP_address_of: - case OP_global_address_of: - case OP_load_data_address: - case OP_load_rodata_address: - case OP_define: - case OP_label: - case OP_jump: - return false; - default: - return true; - } -} - -/* Opcodes whose src1 is a register rather than a width, slot or immediate. */ -bool op_src1_is_reg(opcode_t op) -{ - switch (op) { - case OP_add: - case OP_sub: - case OP_mul: - case OP_div: - case OP_mod: - case OP_lshift: - case OP_rshift: - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: - case OP_eq: - case OP_neq: - case OP_lt: - case OP_leq: - case OP_gt: - case OP_geq: - case OP_write: - case OP_cmov: - return true; - default: - return false; - } -} - -/* Whether src2 names a register. Only a select does: it is the value kept when - * the condition does not hold, and a scan that missed it would take that value - * for dead and drop whatever computed it. - */ -bool op_src2_is_reg(opcode_t op) -{ - return op == OP_cmov; -} - -/* Whether @ir reads @reg as an operand. A 32-bit target names the high half of - * a wide operand in src0_hi or src1_hi, which are -1 when unused. - */ -bool ir_reads_reg(ph2_ir_t *ir, int reg) -{ - if (reg >= 0 && (ir->src0_hi == reg || ir->src1_hi == reg)) - return true; - if (op_src2_is_reg(ir->op) && ir->src2 == reg) - return true; - if (op_src0_is_reg(ir->op) && ir->src0 == reg) - return true; - return op_src1_is_reg(ir->op) && ir->src1 == reg; -} - -/* Main peephole optimization function that applies pattern matching and - * transformation rules to consecutive IR instructions. - * Returns true if any optimization was applied, false otherwise. Drop the - * instructions after @ir through @last, keeping ph2_ir_list.tail on a node - * still in the list. - * - * Every removal in this file goes through here. A bare "ir->next = last->next" - * leaves tail pointing at a removed node, which is why x64-codegen.c used to - * walk a block rather than read its tail. - */ -void ph2_ir_drop_after(basic_block_t *bb, ph2_ir_t *ir, ph2_ir_t *last) -{ - ir->next = last->next; - if (!ir->next) - bb->ph2_ir_list.tail = ir; + return op_property(ir->op, OP_FUSIBLE); } -/* How many argument registers the call @ir reads. A call names none of them. A - * prototyped, non-variadic callee reads only the words its own parameters - * occupy, the aggregate-return address included; any other call, or a call - * through a pointer, may read all of them. - */ -int call_arg_regs(ph2_ir_t *ir) -{ - func_t *callee = ir->op == OP_call ? find_func(ir->func_name) : NULL; - - if (!callee || !callee->has_prototype || callee->va_args) - return MAX_ARGS_IN_REG; - - int words = callee->returns_aggregate ? 1 : 0; - for (int i = 0; i < callee->num_params && words < MAX_ARGS_IN_REG; i++) - words = abi_arg_next(words, &callee->param_defs[i], false); - return words < MAX_ARGS_IN_REG ? words : MAX_ARGS_IN_REG; -} - -/* True when @reg still holds a live value past the end of @bb. - * - * reg_alloc() hands its register file to a successor that this block is the - * only way into (bb_export_regs()), so a register can outlive the block that - * filled it. Scans that stop at the block boundary conclude there according to - * this: with nothing carried, every register dies at the end of a block and an - * unread value is dead. - */ -bool reg_live_out_of_bb(basic_block_t *bb, int reg) -{ - basic_block_t *succs[3]; - - if (!bb || reg < 0 || reg >= REG_CNT) - return false; - - /* A register the allocator pinned holds its variable on every path, so it - * is live out of every block regardless of what the successors record. - */ - if (bb->belong_to && ((bb->belong_to->pinned_regs >> reg) & 1)) - return true; - - succs[0] = bb->next; - succs[1] = bb->then_; - succs[2] = bb->else_; - - for (int i = 0; i < 3; i++) { - if (succs[i] && succs[i]->entry_regs && succs[i]->entry_regs[reg]) - return true; - } - return false; -} - -/* Whether @reg may still be read after @ir, before anything writes it again. - * - * The scan covers the rest of @bb, with a call reading its argument registers. - * Past the end of the block a register keeps its value only when it is pinned, - * or when reg_alloc() handed it to a successor through bb_export_regs(); every - * other successor loads what it reads. - */ -bool reg_read_after(basic_block_t *bb, ph2_ir_t *ir, int reg) +/* Scan local reads and definitions, then consult the successor live set. */ +static bool reg_read_after(basic_block_t *bb, vir_mop_t *ir, int reg) { if (reg < 0) return false; if (reg >= REG_CNT) return true; - - for (ph2_ir_t *p = ir->next; p; p = p->next) { - if (p->op == OP_call || p->op == OP_indirect) { - if (reg < call_arg_regs(p)) - return true; - continue; - } + for (vir_mop_t *p = ir->next; p; p = p->next) { if (ir_reads_reg(p, reg)) return true; - if (op_writes_dest(p->op) && (p->dest == reg || p->dest_hi == reg)) + if (mop_defs(p) & (1u << reg)) return false; } - - return reg_live_out_of_bb(bb, reg); + return (bb->machine_liveout & (1u << reg)) != 0; } /* Whether folding @ir into @last loses a value that is still wanted. The folded @@ -256,7 +39,7 @@ bool reg_read_after(basic_block_t *bb, ph2_ir_t *ir, int reg) * peephole() hands every window with a register pair to pair_insn_fusion(), so * both are single registers here. */ -bool fold_loses_dest(basic_block_t *bb, ph2_ir_t *ir, ph2_ir_t *last) +bool fold_loses_dest(basic_block_t *bb, vir_mop_t *ir, vir_mop_t *last) { return ir->dest != last->dest && reg_read_after(bb, last, ir->dest); } @@ -267,13 +50,13 @@ bool fold_loses_dest(basic_block_t *bb, ph2_ir_t *ir, ph2_ir_t *last) * after the operation, and it cannot be both operands, since the rewrite still * reads the other one. */ -bool const_fold_ok(basic_block_t *bb, ph2_ir_t *li, ph2_ir_t *op) +bool const_fold_ok(basic_block_t *bb, vir_mop_t *li, vir_mop_t *op) { return op->src0 != op->src1 && !fold_loses_dest(bb, li, op); } /* Whether @ir names a 32-bit target's register pair. */ -bool ph2_ir_has_pair(const ph2_ir_t *ir) +bool mop_has_pair(const vir_mop_t *ir) { return ir && (ir->dest_hi >= 0 || ir->src0_hi >= 0 || ir->src1_hi >= 0); } @@ -286,16 +69,16 @@ bool ph2_ir_has_pair(const ph2_ir_t *ir) * result does not overlap an operand, and a low result written first would * otherwise clobber a high operand still to be read. */ -bool pair_insn_fusion(basic_block_t *bb, ph2_ir_t *ph2_ir) +bool pair_insn_fusion(basic_block_t *bb, vir_mop_t *mop) { - ph2_ir_t *next = ph2_ir->next; + vir_mop_t *next = mop->next; - if (!next || next->op != OP_assign || !is_fusible_insn(ph2_ir)) + if (!next || next->op != OP_assign || !is_fusible_insn(mop)) return false; - if (ph2_ir->dest_hi < 0 || ph2_ir->dest != next->src0 || - ph2_ir->dest_hi != next->src0_hi || next->dest_hi < 0) + if (mop->dest_hi < 0 || mop->dest != next->src0 || + mop->dest_hi != next->src0_hi || next->dest_hi < 0) return false; - if (ir_reads_reg(ph2_ir, next->dest) || ir_reads_reg(ph2_ir, next->dest_hi)) + if (ir_reads_reg(mop, next->dest) || ir_reads_reg(mop, next->dest_hi)) return false; /* The fused operation no longer writes its own pair, so that pair must be @@ -303,215 +86,115 @@ bool pair_insn_fusion(basic_block_t *bb, ph2_ir_t *ph2_ir) * copy that is not: "(a & b) ^ (b & a)" became "x = a & b; y = x; x ^ y", * and fusing the move left the XOR reading a stale x. */ - if ((ph2_ir->dest != next->dest && - reg_read_after(bb, next, ph2_ir->dest)) || - (ph2_ir->dest_hi != next->dest_hi && - reg_read_after(bb, next, ph2_ir->dest_hi))) + if ((mop->dest != next->dest && reg_read_after(bb, next, mop->dest)) || + (mop->dest_hi != next->dest_hi && + reg_read_after(bb, next, mop->dest_hi))) return false; - ph2_ir->dest = next->dest; - ph2_ir->dest_hi = next->dest_hi; - ph2_ir_drop_after(bb, ph2_ir, next); + mop->dest = next->dest; + mop->dest_hi = next->dest_hi; + mop_drop_after(bb, mop, next); + return true; +} + +static bool replace_with_move(basic_block_t *bb, + vir_mop_t *ir, + vir_mop_t *next, + int source) +{ + ir->op = OP_assign; + ir->src0 = source; + ir->dest = next->dest; + mop_drop_after(bb, ir, next); return true; } -bool insn_fusion(basic_block_t *bb, ph2_ir_t *ph2_ir) +bool insn_fusion(basic_block_t *bb, vir_mop_t *mop) { - ph2_ir_t *next = ph2_ir->next; + vir_mop_t *next = mop->next; if (!next) return false; - /* ALU instruction fusion. Eliminates redundant move operations following - * arithmetic/logical operations. This is the most fundamental optimization - * that removes temporary register usage. - */ + if ((mop->op == OP_eq || mop->op == OP_neq) && next->op == OP_log_not && + next->src0 == mop->dest && !fold_loses_dest(bb, mop, next)) { + mop->op = mop->op == OP_eq ? OP_neq : OP_eq; + mop->dest = next->dest; + mop_drop_after(bb, mop, next); + return true; + } + + /* Fuse result moves and fold constant identities. */ if (next->op == OP_assign) { - if (is_fusible_insn(ph2_ir) && ph2_ir->dest == next->src0 && - !fold_loses_dest(bb, ph2_ir, next)) { - /* Pattern: {ALU rn, rs1, rs2; mv rd, rn} → {ALU rd, rs1, rs2} - * Example: {add t1, a, b; mv result, t1} → {add result, a, b} - * - * Only when nothing reads rn afterwards. A value with two names is - * copied and then read through the first one as well, and the fused - * instruction no longer writes it. - */ - ph2_ir->dest = next->dest; - ph2_ir_drop_after(bb, ph2_ir, next); + if (is_fusible_insn(mop) && mop->dest == next->src0 && + !fold_loses_dest(bb, mop, next)) { + /* Do not remove a temporary that still has another live use. */ + mop->dest = next->dest; + mop_drop_after(bb, mop, next); return true; } } /* Arithmetic identity with zero constant */ - if (ph2_ir->op == OP_load_constant && ph2_ir->src0 == 0 && - ph2_ir->src1 == 0) { + if (mop->op == OP_load_constant && mop->src0 == 0 && mop->src1 == 0) { if (next->op == OP_add && - (ph2_ir->dest == next->src0 || ph2_ir->dest == next->src1) && - const_fold_ok(bb, ph2_ir, next)) { - /* Pattern: {li 0; add x, 0} → {mov x} (additive identity: x+0 = x) - * Handles both operand positions due to addition commutativity - * Example: {li t1, 0; add result, var, t1} → {mov result, var} - */ + (mop->dest == next->src0 || mop->dest == next->src1) && + const_fold_ok(bb, mop, next)) { int non_zero_src = - (ph2_ir->dest == next->src0) ? next->src1 : next->src0; + (mop->dest == next->src0) ? next->src1 : next->src0; - ph2_ir->op = OP_assign; - ph2_ir->src0 = non_zero_src; - ph2_ir->dest = next->dest; - ph2_ir_drop_after(bb, ph2_ir, next); - return true; + return replace_with_move(bb, mop, next, non_zero_src); } - if (next->op == OP_sub && const_fold_ok(bb, ph2_ir, next)) { - if (ph2_ir->dest == next->src1) { - /* Pattern: {li 0; sub x, 0} → {mov x} (x - 0 = x) - * Example: {li t1, 0; sub result, var, t1} → {mov result, var} - */ - ph2_ir->op = OP_assign; - ph2_ir->src0 = next->src0; - ph2_ir->dest = next->dest; - ph2_ir_drop_after(bb, ph2_ir, next); - return true; + if (next->op == OP_sub && const_fold_ok(bb, mop, next)) { + if (mop->dest == next->src1) { + return replace_with_move(bb, mop, next, next->src0); } - if (ph2_ir->dest == next->src0) { - /* Pattern: {li 0; sub 0, x} → {neg x} (0 - x = -x) - * Example: {li t1, 0; sub result, t1, var} → {neg result, var} - */ - ph2_ir->op = OP_negate; - ph2_ir->src0 = next->src1; - ph2_ir->dest = next->dest; - ph2_ir_drop_after(bb, ph2_ir, next); + if (mop->dest == next->src0) { + mop->op = OP_negate; + mop->src0 = next->src1; + mop->dest = next->dest; + mop_drop_after(bb, mop, next); return true; } } if (next->op == OP_mul && - (ph2_ir->dest == next->src0 || ph2_ir->dest == next->src1) && - const_fold_ok(bb, ph2_ir, next)) { - /* Pattern: {li 0; mul x, 0} → {li 0} (absorbing element: x * 0 = 0) - * Example: {li t1, 0; mul result, var, t1} → {li result, 0} - * Eliminates multiplication entirely - */ - ph2_ir->op = OP_load_constant; - ph2_ir->src0 = 0; - ph2_ir->dest = next->dest; - ph2_ir_drop_after(bb, ph2_ir, next); + (mop->dest == next->src0 || mop->dest == next->src1) && + const_fold_ok(bb, mop, next)) { + mop->op = OP_load_constant; + mop->src0 = 0; + mop->dest = next->dest; + mop_drop_after(bb, mop, next); return true; } } /* Multiplicative identity with one constant */ - if (ph2_ir->op == OP_load_constant && ph2_ir->src0 == 1 && - ph2_ir->src1 == 0) { + if (mop->op == OP_load_constant && mop->src0 == 1 && mop->src1 == 0) { if (next->op == OP_mul && - (ph2_ir->dest == next->src0 || ph2_ir->dest == next->src1) && - const_fold_ok(bb, ph2_ir, next)) { - /* Pattern: {li 1; mul x, 1} → {mov x} (multiplicative identity: x * - * 1 = x) Example: {li t1, 1; mul result, var, t1} → {mov result, - * var} Handles both operand positions due to multiplication - * commutativity - */ - ph2_ir->op = OP_assign; - ph2_ir->src0 = ph2_ir->dest == next->src0 ? next->src1 : next->src0; - ph2_ir->dest = next->dest; - ph2_ir_drop_after(bb, ph2_ir, next); - return true; + (mop->dest == next->src0 || mop->dest == next->src1) && + const_fold_ok(bb, mop, next)) { + return replace_with_move( + bb, mop, next, + mop->dest == next->src0 ? next->src1 : next->src0); } } /* Bitwise identity operations. src1 is the constant's high word, which an * eight-byte operation includes: 0xffffffffULL is no identity for it. */ - if (ph2_ir->op == OP_load_constant && ph2_ir->src0 == -1 && - ph2_ir->src1 == (next->size_bytes > 4 ? -1 : 0) && - next->op == OP_bit_and && ph2_ir->dest == next->src1 && - const_fold_ok(bb, ph2_ir, next)) { - /* Pattern: {li -1; and x, -1} → {mov x} (x & 0xFFFFFFFF = x) Example: - * {li t1, -1; and result, var, t1} → {mov result, var} Eliminates - * bitwise AND with all-ones mask - */ - ph2_ir->op = OP_assign; - ph2_ir->src0 = next->src0; - ph2_ir->dest = next->dest; - ph2_ir_drop_after(bb, ph2_ir, next); - return true; - } - - if (ph2_ir->op == OP_load_constant && ph2_ir->src0 == 0 && - ph2_ir->src1 == 0 && (next->op == OP_lshift || next->op == OP_rshift) && - ph2_ir->dest == next->src1 && const_fold_ok(bb, ph2_ir, next)) { - /* Pattern: {li 0; shl/shr x, 0} → {mov x} (x << 0 = x >> 0 = x) - * Example: {li t1, 0; shl result, var, t1} → {mov result, var} - * Eliminates no-op shift operations - */ - ph2_ir->op = OP_assign; - ph2_ir->src0 = next->src0; - ph2_ir->dest = next->dest; - ph2_ir_drop_after(bb, ph2_ir, next); - return true; - } - - if (ph2_ir->op == OP_load_constant && ph2_ir->src0 == 0 && - ph2_ir->src1 == 0 && next->op == OP_bit_or && - ph2_ir->dest == next->src1 && const_fold_ok(bb, ph2_ir, next)) { - /* Pattern: {li 0; or x, 0} → {mov x} (x | 0 = x) Example: {li t1, 0; or - * result, var, t1} → {mov result, var} Eliminates bitwise OR with zero - * (identity element) - */ - ph2_ir->op = OP_assign; - ph2_ir->src0 = next->src0; - ph2_ir->dest = next->dest; - ph2_ir_drop_after(bb, ph2_ir, next); - return true; - } - - /* Power-of-2 multiplication to shift conversion. Shift operations are - * significantly faster than multiplication - */ - if (ph2_ir->op == OP_load_constant && ph2_ir->src0 > 0 && - ph2_ir->src1 == 0 && next->op == OP_mul && ph2_ir->dest == next->src1) { - int shift_amount = exact_log2(ph2_ir->src0); - if (shift_amount >= 0 && const_fold_ok(bb, ph2_ir, next)) { - /* Pattern: {li 2^n; mul x, 2^n} → {li n; shl x, n} Example: {li t1, - * 4; mul result, var, t1} → - * {li t1, 2; shl result, var, t1} - */ - ph2_ir->src0 = shift_amount; - next->op = OP_lshift; - return true; - } - } - - /* XOR identity operation */ - if (ph2_ir->op == OP_load_constant && ph2_ir->src0 == 0 && - ph2_ir->src1 == 0 && next->op == OP_bit_xor && - ph2_ir->dest == next->src1 && const_fold_ok(bb, ph2_ir, next)) { - /* Pattern: {li 0; xor x, 0} → {mov x} (x ^ 0 = x) Example: {li t1, 0; - * xor result, var, t1} → {mov result, var} Completes bitwise identity - * optimization coverage - */ - ph2_ir->op = OP_assign; - ph2_ir->src0 = next->src0; - ph2_ir->dest = next->dest; - ph2_ir_drop_after(bb, ph2_ir, next); - return true; + if (mop->op == OP_load_constant && mop->src0 == -1 && + mop->src1 == (next->size_bytes > 4 ? -1 : 0) && + next->op == OP_bit_and && mop->dest == next->src1 && + const_fold_ok(bb, mop, next)) { + return replace_with_move(bb, mop, next, next->src0); } - /* Extended multiplicative identity (operand position variant) Handles the - * case where constant 1 is in src0 position of multiplication - */ - if (ph2_ir->op == OP_load_constant && ph2_ir->src0 == 1 && - ph2_ir->src1 == 0 && next->op == OP_mul && ph2_ir->dest == next->src0 && - const_fold_ok(bb, ph2_ir, next)) { - /* Pattern: {li 1; mul 1, x} → {mov x} (1 * x = x) Example: {li t1, 1; - * mul result, t1, var} → {mov result, var} Covers multiplication - * commutativity edge case - */ - ph2_ir->op = OP_assign; - ph2_ir->src0 = next->src1; - ph2_ir->dest = next->dest; - ph2_ir_drop_after(bb, ph2_ir, next); - return true; - } + if (mop->op == OP_load_constant && mop->src0 == 0 && mop->src1 == 0 && + (next->op == OP_lshift || next->op == OP_rshift || + next->op == OP_bit_or || next->op == OP_bit_xor) && + mop->dest == next->src1 && const_fold_ok(bb, mop, next)) + return replace_with_move(bb, mop, next, next->src0); return false; } @@ -519,12 +202,20 @@ bool insn_fusion(basic_block_t *bb, ph2_ir_t *ph2_ir) /* Whether @ir does nothing but set its destination register: a move, a slot * load or a constant load. */ -bool is_plain_register_def(const ph2_ir_t *ir) +bool is_plain_register_def(const vir_mop_t *ir) { return ir->op == OP_assign || ir->op == OP_load || ir->op == OP_global_load || ir->op == OP_load_constant; } +static bool is_full_register_copy(const vir_mop_t *ir) +{ + return ir && ir->op == OP_assign && ir->dest_hi < 0 && ir->src0_hi < 0 && + !(ir->is_unsigned && ir->size_bytes < PTR_SIZE) && + ir->dest != ir->src0 && ir->dest >= 0 && ir->dest < REG_CNT && + ir->src0 >= 0 && ir->src0 < REG_CNT; +} + /* Redundant move elimination: a move, load or constant load whose register is * immediately overwritten by another one is dead. {mov rd, rs1; mov rd, rs2}, * {load rd, ofs; mov rd, rs}, {li rd, imm; load rd, ofs} and the other @@ -536,23 +227,106 @@ bool is_plain_register_def(const ph2_ir_t *ir) * overwritten unsigned int constant kept only its low 32 bits. A constant's * high word in src1 was likewise dropped. */ -bool redundant_move_elim(basic_block_t *bb, ph2_ir_t *ph2_ir) +bool redundant_move_elim(basic_block_t *bb, vir_mop_t *mop) { - ph2_ir_t *next = ph2_ir->next; + vir_mop_t *next = mop->next; if (!next) return false; /* A volatile load is an access the program performs, not only a value it * computes, so it stays even when its register is overwritten unread. */ - if (!is_plain_register_def(ph2_ir) || !is_plain_register_def(next) || - ph2_ir->is_volatile || ph2_ir->dest != next->dest || - (ph2_ir->dest_hi >= 0 && ph2_ir->dest_hi != next->dest_hi) || - ir_reads_reg(next, ph2_ir->dest) || ir_reads_reg(next, ph2_ir->dest_hi)) + if (!is_plain_register_def(mop) || !is_plain_register_def(next) || + mop->is_volatile || mop->dest != next->dest || + (mop->dest_hi >= 0 && mop->dest_hi != next->dest_hi) || + ir_reads_reg(next, mop->dest) || ir_reads_reg(next, mop->dest_hi)) return false; - memcpy(ph2_ir, next, sizeof(ph2_ir_t)); - ph2_ir_drop_after(bb, ph2_ir, ph2_ir); + memcpy(mop, next, sizeof(vir_mop_t)); + mop_drop_after(bb, mop, mop); + return true; +} + +/* {t = r; op ..., t, ...} -> {op ..., r, ...} when t dies there. A post + * increment ("i++" keeps i's old value in a temporary) and argument staging + * leave such copies in every loop; forwarding the source drops the move. + * + * Only a plain full-width copy qualifies: a register pair, or an unsigned + * narrowing copy that x86-64 emits as a zero extension, changes the value. + */ +bool copy_forward(basic_block_t *bb, vir_mop_t *mop) +{ + vir_mop_t *next = mop->next; + int t = mop->dest; + int r = mop->src0; + bool used = false; + + if (!next || !is_full_register_copy(mop)) + return false; + if (next->src0_hi >= 0 || next->src1_hi >= 0 || next->dest_hi >= 0) + return false; + + /* Only instructions that read exactly the registers they name. A call reads + * its argument registers without naming them, and several opcodes keep + * something other than a register in src0; the copy may stage an argument, + * or its number may merely match an unrelated field. + */ + if (!op_property(next->op, OP_COPY_FORWARD_SAFE)) + return false; + /* The copy has to die here, unless the instruction itself rewrites it. */ + if (!(op_writes_dest(next->op) && next->dest == t) && + reg_read_after(bb, next, t)) + return false; + + /* Backends lower "rd = op a, b" as "mov rd, a; op rd, b", so an instruction + * that writes r may read r only as its first operand: "t = r; r = sub a, t" + * must not become "r = sub a, r". + */ + if (op_writes_dest(next->op) && next->dest == r && + op_src1_is_reg(next->op) && next->src1 == t) + return false; + if (op_src0_is_reg(next->op) && next->src0 == t) { + next->src0 = r; + used = true; + } + if (op_src1_is_reg(next->op) && next->src1 == t) { + next->src1 = r; + used = true; + } + if (!used) + return false; + memcpy(mop, next, sizeof(vir_mop_t)); + mop_drop_after(bb, mop, mop); + return true; +} + +/* {t = r; li r, K; [t] = r} -> {li t, K; [r] = t} when both die at the store. + * The copy exists only because the value was loaded into the register the + * address was in; swapping the two roles drops it, and leaves the address + * feeding the store directly, where a backend can fold it into the access. + */ +bool store_copy_swap(basic_block_t *bb, vir_mop_t *mop) +{ + vir_mop_t *li = mop->next; + vir_mop_t *wr = li ? li->next : NULL; + int t = mop->dest; + int r = mop->src0; + + if (!wr || !is_full_register_copy(mop)) + return false; + if (li->op != OP_load_constant || li->dest != r || li->dest_hi >= 0) + return false; + if (wr->op != OP_write || wr->src0 != t || wr->src1 != r || + wr->src0_hi >= 0 || wr->src1_hi >= 0) + return false; + if (reg_read_after(bb, wr, t) || reg_read_after(bb, wr, r)) + return false; + + memcpy(mop, li, sizeof(vir_mop_t)); + mop->dest = t; + wr->src0 = r; + wr->src1 = t; + mop_drop_after(bb, mop, mop); return true; } @@ -560,9 +334,9 @@ bool redundant_move_elim(basic_block_t *bb, ph2_ir_t *ph2_ir) * loads and dead stores that access the same memory location. Conservative * implementation to maintain bootstrap stability. */ -bool eliminate_load_store_pairs(basic_block_t *bb, ph2_ir_t *ph2_ir) +bool eliminate_load_store_pairs(basic_block_t *bb, vir_mop_t *mop) { - ph2_ir_t *next = ph2_ir->next; + vir_mop_t *next = mop->next; if (!next) return false; @@ -572,7 +346,7 @@ bool eliminate_load_store_pairs(basic_block_t *bb, ph2_ir_t *ph2_ir) * store [addr], val2} → {store [addr], val2} First store is dead if * immediately overwritten */ - if (ph2_ir->op == OP_store && next->op == OP_store) { + if (mop->op == OP_store && next->op == OP_store) { /* Same slot written twice in a row: only the second is observable. * Rewrite this one into the second and drop it, which keeps the list * links the caller is holding valid. @@ -581,15 +355,14 @@ bool eliminate_load_store_pairs(basic_block_t *bb, ph2_ir_t *ph2_ir) * same slot leaves the bytes the second does not cover holding what the * first put there, so dropping the first loses them. */ - if (ph2_ir->src1 == next->src1 && ph2_ir->src1 >= 0 && - !ph2_ir->is_volatile && !next->is_volatile && - ph2_ir->size_bytes == next->size_bytes && - ph2_ir->is_pointer == next->is_pointer && - ph2_ir->ofs_based_on_stack_top == next->ofs_based_on_stack_top) { - ph2_ir->src0 = next->src0; - ph2_ir->size_bytes = next->size_bytes; - ph2_ir->is_pointer = next->is_pointer; - ph2_ir_drop_after(bb, ph2_ir, next); + if (mop->src1 == next->src1 && mop->src1 >= 0 && !mop->is_volatile && + !next->is_volatile && mop->size_bytes == next->size_bytes && + mop->is_pointer == next->is_pointer && + mop->ofs_based_on_stack_top == next->ofs_based_on_stack_top) { + mop->src0 = next->src0; + mop->size_bytes = next->size_bytes; + mop->is_pointer = next->is_pointer; + mop_drop_after(bb, mop, next); return true; } } @@ -600,21 +373,21 @@ bool eliminate_load_store_pairs(basic_block_t *bb, ph2_ir_t *ph2_ir) if (next->is_volatile) return false; - /* Pattern 2: Redundant consecutive loads from same local location {load - * rd1, [addr]; load rd2, [addr]} → {load rd1, [addr]; mov rd2, rd1} Second - * load can reuse the first load's result Only apply if addresses are simple - * (not complex expressions) + /* Reuse only an identical load. Width, extension and frame origin affect + * the loaded value even when the encoded offsets match. */ - if (ph2_ir->op == OP_load && next->op == OP_load) { - /* Check if loading from same memory location */ - if (ph2_ir->src0 == next->src0 && ph2_ir->src1 == next->src1 && - ph2_ir->src0 >= 0 && ph2_ir->src1 >= 0) { - /* Replace second load with move */ - next->op = OP_assign; - next->src0 = ph2_ir->dest; /* Result of first load */ - next->src1 = 0; - return true; - } + if (mop->op == next->op && + (mop->op == OP_load || mop->op == OP_global_load) && + mop->src0 == next->src0 && mop->src1 == next->src1 && + (mop->op == OP_global_load || (mop->src0 >= 0 && mop->src1 >= 0)) && + mop->size_bytes == next->size_bytes && + mop->is_unsigned == next->is_unsigned && + mop->is_pointer == next->is_pointer && + mop->ofs_based_on_stack_top == next->ofs_based_on_stack_top) { + next->op = OP_assign; + next->src0 = mop->dest; + next->src1 = 0; + return true; } /* Pattern 3: Store followed by load from same location (store-to-load @@ -626,14 +399,14 @@ bool eliminate_load_store_pairs(basic_block_t *bb, ph2_ir_t *ph2_ir) * only at equal width, since a store narrower than the slot leaves the load * sign-extending fewer bits than the register holds. */ - if (ph2_ir->op == OP_store && next->op == OP_load) { - if (ph2_ir->src1 == next->src0 && ph2_ir->src0 >= 0 && - ph2_ir->size_bytes == next->size_bytes && - ph2_ir->is_pointer == next->is_pointer && - ph2_ir->size_bytes >= PTR_SIZE && - ph2_ir->ofs_based_on_stack_top == next->ofs_based_on_stack_top) { + if (mop->op == OP_store && next->op == OP_load) { + if (mop->src1 == next->src0 && mop->src0 >= 0 && + mop->size_bytes == next->size_bytes && + mop->is_pointer == next->is_pointer && + mop->size_bytes >= PTR_SIZE && + mop->ofs_based_on_stack_top == next->ofs_based_on_stack_top) { next->op = OP_assign; - next->src0 = ph2_ir->src0; /* the register that was stored */ + next->src0 = mop->src0; /* the register that was stored */ next->src1 = 0; return true; } @@ -647,37 +420,29 @@ bool eliminate_load_store_pairs(basic_block_t *bb, ph2_ir_t *ph2_ir) * sign-extended byte and the store writes that extension over bytes the * load never read. */ - if (ph2_ir->op == OP_load && next->op == OP_store) { - if (ph2_ir->dest == next->src0 && ph2_ir->src0 == next->src1 && - ph2_ir->src0 >= 0 && ph2_ir->size_bytes == next->size_bytes && - ph2_ir->is_pointer == next->is_pointer && - ph2_ir->ofs_based_on_stack_top == next->ofs_based_on_stack_top) { - ph2_ir_drop_after(bb, ph2_ir, next); + if (mop->op == OP_load && next->op == OP_store) { + if (mop->dest == next->src0 && mop->src0 == next->src1 && + mop->src0 >= 0 && mop->size_bytes == next->size_bytes && + mop->is_pointer == next->is_pointer && + mop->ofs_based_on_stack_top == next->ofs_based_on_stack_top) { + mop_drop_after(bb, mop, next); return true; } } /* Pattern 5: Global store/load optimizations (carefully enabled) */ - if (ph2_ir->op == OP_global_store && next->op == OP_global_store) { + if (mop->op == OP_global_store && next->op == OP_global_store) { /* Consecutive global stores to same location */ - if (ph2_ir->src0 == next->src0 && ph2_ir->src1 == next->src1) { + if (mop->src0 == next->src0 && mop->src1 == next->src1 && + mop->size_bytes == next->size_bytes && !mop->is_volatile && + !next->is_volatile) { /* Remove first store - it's dead */ - ph2_ir->dest = next->dest; - ph2_ir_drop_after(bb, ph2_ir, next); + mop->dest = next->dest; + mop_drop_after(bb, mop, next); return true; } } - if (ph2_ir->op == OP_global_load && next->op == OP_global_load) { - /* Consecutive global loads from same location */ - if (ph2_ir->src0 == next->src0 && ph2_ir->src1 == next->src1) { - /* Replace second load with move */ - next->op = OP_assign; - next->src0 = ph2_ir->dest; - next->src1 = 0; - return true; - } - } return false; } @@ -704,15 +469,15 @@ bool eliminate_load_store_pairs(basic_block_t *bb, ph2_ir_t *ph2_ir) * * Returns true if optimization was applied */ -bool strength_reduction(basic_block_t *bb, ph2_ir_t *ph2_ir) +bool strength_reduction(basic_block_t *bb, vir_mop_t *mop) { - if (!ph2_ir || !ph2_ir->next) + if (!mop || !mop->next) return false; - ph2_ir_t *next = ph2_ir->next; + vir_mop_t *next = mop->next; /* Check for constant load followed by division or modulo */ - if (ph2_ir->op != OP_load_constant) + if (mop->op != OP_load_constant) return false; /* Each rewrite below loads a different constant into the register, which @@ -720,17 +485,17 @@ bool strength_reduction(basic_block_t *bb, ph2_ir_t *ph2_ir) */ if (next->op != OP_div && next->op != OP_mod && next->op != OP_mul) return false; - if (next->src0 != ph2_ir->dest && next->src1 != ph2_ir->dest) + if (next->src0 != mop->dest && next->src1 != mop->dest) return false; - int value = ph2_ir->src0; + int value = mop->src0; /* Check if value is a power of 2. src1 is the high word of an eight-byte * constant, and 0x100000004ULL is no power of two. */ - if (ph2_ir->src1 || value <= 0 || (value & (value - 1)) != 0) + if (mop->src1 || value <= 0 || (value & (value - 1)) != 0) return false; - if (!const_fold_ok(bb, ph2_ir, next)) + if (!const_fold_ok(bb, mop, next)) return false; /* Calculate shift amount for power of 2 */ @@ -745,18 +510,18 @@ bool strength_reduction(basic_block_t *bb, ph2_ir_t *ph2_ir) * unsigned) */ if (next->op == OP_div && next->src0_is_unsigned && - next->src1 == ph2_ir->dest) { + next->src1 == mop->dest) { /* Convert division to right shift */ - ph2_ir->src0 = shift; /* Load shift amount instead */ + mop->src0 = shift; /* Load shift amount instead */ next->op = OP_rshift; return true; } /* Pattern 2: Modulo by power of 2 → bitwise AND x % 2^n = x & (2^n - 1) */ if (next->op == OP_mod && next->src0_is_unsigned && - next->src1 == ph2_ir->dest) { + next->src1 == mop->dest) { /* Convert modulo to bitwise AND */ - ph2_ir->src0 = value - 1; /* Load mask (2^n - 1) */ + mop->src0 = value - 1; /* Load mask (2^n - 1) */ next->op = OP_bit_and; return true; } @@ -765,16 +530,16 @@ bool strength_reduction(basic_block_t *bb, ph2_ir_t *ph2_ir) * x * 2^n = x << n */ if (next->op == OP_mul) { - if (next->src0 == ph2_ir->dest) { + if (next->src0 == mop->dest) { /* 2^n * x = x << n */ - ph2_ir->src0 = shift; /* Load shift amount */ + mop->src0 = shift; /* Load shift amount */ next->op = OP_lshift; - next->src0 = next->src1; /* Move x to src0 */ - next->src1 = ph2_ir->dest; /* Shift amount in src1 */ + next->src0 = next->src1; /* Move x to src0 */ + next->src1 = mop->dest; /* Shift amount in src1 */ return true; - } else if (next->src1 == ph2_ir->dest) { + } else if (next->src1 == mop->dest) { /* x * 2^n = x << n */ - ph2_ir->src0 = shift; /* Load shift amount */ + mop->src0 = shift; /* Load shift amount */ next->op = OP_lshift; return true; } @@ -788,23 +553,19 @@ bool strength_reduction(basic_block_t *bb, ph2_ir_t *ph2_ir) * * Returns true when it rewrote something. */ -bool bitwise_optimization(basic_block_t *bb, ph2_ir_t *ph2_ir) +bool bitwise_optimization(basic_block_t *bb, vir_mop_t *mop) { - if (!ph2_ir || !ph2_ir->next) + if (!mop || !mop->next) return false; - ph2_ir_t *next = ph2_ir->next; + vir_mop_t *next = mop->next; /* Pattern 1: Double complement → identity ~(~x) = x. The first complement * no longer writes its register, so nothing else may read it. */ - if (ph2_ir->op == OP_bit_not && next->op == OP_bit_not && - next->src0 == ph2_ir->dest && !fold_loses_dest(bb, ph2_ir, next)) { - /* Replace with simple assignment */ - ph2_ir->op = OP_assign; - ph2_ir->dest = next->dest; - ph2_ir_drop_after(bb, ph2_ir, next); - return true; + if (mop->op == OP_bit_not && next->op == OP_bit_not && + next->src0 == mop->dest && !fold_loses_dest(bb, mop, next)) { + return replace_with_move(bb, mop, next, mop->src0); } /* Pattern 2: AND with zero → zero x & 0 = 0, and OR with all-ones → @@ -813,15 +574,14 @@ bool bitwise_optimization(basic_block_t *bb, ph2_ir_t *ph2_ir) * src1, all ones as well. The identities x & -1, x | 0, x ^ 0 and x << 0 * belong to insn_fusion(), which tries them first. */ - if (ph2_ir->op == OP_load_constant && - ((ph2_ir->src0 == 0 && ph2_ir->src1 == 0 && next->op == OP_bit_and) || - (ph2_ir->src0 == -1 && - ph2_ir->src1 == (next->size_bytes > 4 ? -1 : 0) && + if (mop->op == OP_load_constant && + ((mop->src0 == 0 && mop->src1 == 0 && next->op == OP_bit_and) || + (mop->src0 == -1 && mop->src1 == (next->size_bytes > 4 ? -1 : 0) && next->op == OP_bit_or)) && - (next->src0 == ph2_ir->dest || next->src1 == ph2_ir->dest) && - const_fold_ok(bb, ph2_ir, next)) { - ph2_ir->dest = next->dest; - ph2_ir_drop_after(bb, ph2_ir, next); + (next->src0 == mop->dest || next->src1 == mop->dest) && + const_fold_ok(bb, mop, next)) { + mop->dest = next->dest; + mop_drop_after(bb, mop, next); return true; } @@ -832,29 +592,28 @@ bool bitwise_optimization(basic_block_t *bb, ph2_ir_t *ph2_ir) * are more complex but offer significant optimization opportunities Returns * true if optimization was applied */ -bool triple_pattern_optimization(basic_block_t *bb, ph2_ir_t *ph2_ir) +bool triple_pattern_optimization(basic_block_t *bb, vir_mop_t *mop) { - if (!ph2_ir || !ph2_ir->next || !ph2_ir->next->next) + if (!mop || !mop->next || !mop->next->next) return false; - ph2_ir_t *second = ph2_ir->next; - ph2_ir_t *third = second->next; + vir_mop_t *second = mop->next; + vir_mop_t *third = second->next; /* Pattern 1: Store-load-store elimination {store val1, addr; load r, addr; * store val2, addr} The middle load is pointless if not used elsewhere */ - if (ph2_ir->op == OP_store && second->op == OP_load && - !second->is_volatile && third->op == OP_store && - ph2_ir->src1 == second->src0 && /* same address */ - ph2_ir->dest == second->src1 && /* same offset */ - second->src0 == third->src1 && /* same address */ - second->src1 == third->dest) { /* same offset */ + if (mop->op == OP_store && second->op == OP_load && !second->is_volatile && + third->op == OP_store && mop->src1 == second->src0 && /* same address */ + mop->dest == second->src1 && /* same offset */ + second->src0 == third->src1 && /* same address */ + second->src1 == third->dest) { /* same offset */ /* Only when nothing reads the loaded value, the third store included: * without the load the register keeps what it held before. */ if (!reg_read_after(bb, second, second->dest)) { /* The load result is not used, can eliminate it */ - ph2_ir->next = third; + mop->next = third; return true; } } @@ -862,16 +621,16 @@ bool triple_pattern_optimization(basic_block_t *bb, ph2_ir_t *ph2_ir) /* Pattern 2: Consecutive stores to same location {store v1, addr; store v2, * addr; store v3, addr} Only the last store matters */ - if (ph2_ir->op == OP_store && second->op == OP_store && - third->op == OP_store && !ph2_ir->is_volatile && !second->is_volatile && - !third->is_volatile && ph2_ir->src1 == second->src1 && - ph2_ir->dest == second->dest && second->src1 == third->src1 && + if (mop->op == OP_store && second->op == OP_store && + third->op == OP_store && !mop->is_volatile && !second->is_volatile && + !third->is_volatile && mop->src1 == second->src1 && + mop->dest == second->dest && second->src1 == third->src1 && second->dest == third->dest) { /* All three stores go to the same location Only the last one matters, * eliminate first two */ - ph2_ir->src0 = third->src0; /* Use last value */ - ph2_ir_drop_after(bb, ph2_ir, third); /* Skip middle stores */ + mop->src0 = third->src0; /* Use last value */ + mop_drop_after(bb, mop, third); /* Skip middle stores */ return true; } @@ -905,12 +664,41 @@ bool triple_pattern_optimization(basic_block_t *bb, ph2_ir_t *ph2_ir) return false; } +/* Apply the first rule that matches the window starting at @ir, and report + * whether one did. Every rewrite leaves @ir in place and changes what follows + * it, so the caller tries the same position again. + */ +bool peephole_at(basic_block_t *bb, vir_mop_t *ir) +{ + vir_mop_t *next = ir->next; + if (!next) + return false; + + /* Remove moves to themselves and unused scalar literals. */ + if ((next->op == OP_assign && next->dest == next->src0 && + next->dest_hi == next->src0_hi) || + (next->op == OP_load_constant && next->dest_hi < 0 && + !reg_read_after(bb, next, next->dest))) { + mop_drop_after(bb, ir, next); + return true; + } + + if (PTR_SIZE < 8 && + (mop_has_pair(ir) || mop_has_pair(next) || mop_has_pair(next->next))) + return pair_insn_fusion(bb, ir); + + return triple_pattern_optimization(bb, ir) || insn_fusion(bb, ir) || + strength_reduction(bb, ir) || bitwise_optimization(bb, ir) || + store_copy_swap(bb, ir) || copy_forward(bb, ir) || + redundant_move_elim(bb, ir) || eliminate_load_store_pairs(bb, ir); +} + /* Main peephole optimization driver. * - * This runs on ph2_ir_t, after register allocation, and so sees only what + * This runs on vir_mop_t, after register allocation, and so sees only what * assigning registers makes visible. Constant folding, common subexpression - * elimination and dead code elimination have already run over insn_t in the SSA - * optimizer and are not repeated here. + * elimination and dead code elimination have already run over VIR and are not + * repeated here. * * What is left to do at this level: * - self-assignment elimination, for assignments allocation itself created @@ -921,72 +709,18 @@ bool triple_pattern_optimization(basic_block_t *bb, ph2_ir_t *ph2_ir) */ void peephole(void) { - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - /* Local peephole optimizations on post-register-allocation IR */ + FOR_EACH_FUNCTION_BODY(func) + { + /* Local peephole optimizations on post-register-allocation IR. A + * rewrite can expose another at the same position -- a dead load + * dropped in front of a copy leaves the copy there to be forwarded -- + * so each position is retried, a bounded number of times in case a rule + * reports a change that does not shrink the window. + */ for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (ph2_ir_t *ir = bb->ph2_ir_list.head; ir; ir = ir->next) { - ph2_ir_t *next = ir->next; - if (!next) - continue; - - /* Self-assignment elimination Keep this as a safety net: SSA - * handles most cases, but register allocation might create new - * self-assignments - */ - if (next->op == OP_assign && next->dest == next->src0 && - next->dest_hi == next->src0_hi) { - ph2_ir_drop_after(bb, ir, next); - continue; - } - - /* Every rewrite below moves this instruction's result to the - * destination of the one after it, dropping the write to the - * register it named. That is fine for a temporary, whose value - * nothing wants again, and wrong for a pinned register: a - * variable lives there for the whole function and nothing else - * ever reloads it, so "li rbx, 0; add rax, rsi, rbx" must not - * become "mov rax, rsi" and leave rbx unwritten. - */ - if (ir->dest >= 0 && ir->dest < REG_CNT && - ((func->pinned_regs >> ir->dest) & 1)) - continue; - - if (PTR_SIZE < 8 && - (ph2_ir_has_pair(ir) || ph2_ir_has_pair(next) || - ph2_ir_has_pair(next->next))) { - pair_insn_fusion(bb, ir); - continue; - } - - /* Try triple pattern optimization first (3-instruction - * sequences) - */ - if (triple_pattern_optimization(bb, ir)) - continue; - - /* Try instruction fusion (2-instruction sequences) */ - if (insn_fusion(bb, ir)) - continue; - - /* Apply strength reduction for power-of-2 operations */ - if (strength_reduction(bb, ir)) - continue; - - /* Apply bitwise operation optimizations */ - if (bitwise_optimization(bb, ir)) - continue; - - /* Apply redundant move elimination */ - if (redundant_move_elim(bb, ir)) - continue; - - /* Apply load/store elimination */ - if (eliminate_load_store_pairs(bb, ir)) - continue; + for (vir_mop_t *ir = bb->mops.head; ir; ir = ir->next) { + for (int tries = 0; tries < 4 && peephole_at(bb, ir); tries++) + ; } } } diff --git a/src/preprocessor.c b/src/preprocessor.c index 27bfe5f6..c82a3ed2 100644 --- a/src/preprocessor.c +++ b/src/preprocessor.c @@ -76,6 +76,30 @@ token_t *copy_token(const token_t *tk) return new_tk; } +/* Move the token stream the parser reads into a fresh arena and drop the old + * one. Preprocessing lexes every included file and copies every token a macro + * expands, so the old arena holds about three dead tokens for each one that + * reaches the parser, along with macros, hide sets and conditional-inclusion + * records. A token owns nothing else in it: literals are interned, and the + * source buffers that diagnostics quote live in SRC_FILE_MAP. + */ +token_t *compact_token_stream(token_t *head) +{ + arena_t *preprocessed = TOKEN_ARENA; + token_t first; + token_t *tail = &first; + + TOKEN_ARENA = arena_init(LARGE_ARENA_SIZE); + for (token_t *tk = head; tk; tk = tk->next) { + tail->next = copy_token(tk); + tail = tail->next; + } + tail->next = NULL; + mark_token_stream_end(); + retire_token_arena(preprocessed); + return first.next; +} + typedef struct macro { char *name; int param_num; @@ -89,6 +113,14 @@ typedef struct macro { token_t *(*handler)(token_t *); } macro_t; +static macro_t *new_macro(char *name) +{ + macro_t *macro = arena_calloc(TOKEN_ARENA, 1, sizeof(*macro)); + + macro->name = name; + return macro; +} + bool is_macro_defined(char *name) { const macro_t *macro = hashmap_get(MACROS, name); @@ -115,13 +147,11 @@ void define_builtin_object_macro(const char *name, token_kind_t kind, const char *replacement) { - macro_t *macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - - macro->name = intern_string((char *) name); + macro_t *macro = new_macro(intern_string((char *) name)); macro->replacement = new_token(kind, &synth_built_in_loc, strlen(replacement)); macro->replacement->literal = intern_string((char *) replacement); - hashmap_put(MACROS, macro->name, macro); + hashmap_put_borrowed(MACROS, macro->name, macro); } /* Integer minimum macros are preprocessing token sequences rather than one @@ -130,17 +160,16 @@ void define_builtin_object_macro(const char *name, */ void define_builtin_negative_macro(const char *name, const char *magnitude) { - macro_t *macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); + macro_t *macro = new_macro(intern_string((char *) name)); token_t *minus = new_token(T_minus, &synth_built_in_loc, 1); token_t *numeric = new_token(T_numeric, &synth_built_in_loc, strlen(magnitude)); - macro->name = intern_string((char *) name); minus->literal = "-"; numeric->literal = intern_string((char *) magnitude); minus->next = numeric; macro->replacement = minus; - hashmap_put(MACROS, macro->name, macro); + hashmap_put_borrowed(MACROS, macro->name, macro); } token_t *append_builtin_macro_token(token_t **replacement, @@ -148,6 +177,38 @@ token_t *append_builtin_macro_token(token_t **replacement, token_kind_t kind, const char *literal); +typedef struct { + token_kind_t kind; + const char *literal; +} builtin_macro_token_t; + +static void append_builtin_macro_tokens(token_t **replacement, + token_t **tail, + const builtin_macro_token_t *tokens, + unsigned int count) +{ + for (unsigned int i = 0; i < count; i++) + append_builtin_macro_token(replacement, tail, tokens[i].kind, + tokens[i].literal); +} + +static macro_t *new_builtin_function_macro(const char *name, + const char *first_param, + const char *second_param) +{ + macro_t *macro = new_macro(intern_string((char *) name)); + const char *params[] = {first_param, second_param}; + + macro->is_function_like = true; + macro->param_num = second_param ? 2 : 1; + for (int i = 0; i < macro->param_num; i++) { + macro->param_names[i] = + new_token(T_identifier, &synth_built_in_loc, strlen(params[i])); + macro->param_names[i]->literal = intern_string((char *) params[i]); + } + return macro; +} + /* C99's integer construction macros append a target representation suffix to * their single integer-token argument. The preprocessor's ordinary ## path * rescans the joined spelling, so the replacement remains a numeric token. @@ -155,10 +216,9 @@ token_t *append_builtin_macro_token(token_t **replacement, void define_builtin_integer_construction_macro(const char *name, const char *suffix) { - macro_t *macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); + macro_t *macro = new_macro(intern_string((char *) name)); token_t *tail = NULL; - macro->name = intern_string((char *) name); macro->is_function_like = true; macro->param_num = 1; macro->param_names[0] = new_token(T_identifier, &synth_built_in_loc, 5); @@ -171,7 +231,7 @@ void define_builtin_integer_construction_macro(const char *name, append_builtin_macro_token(¯o->replacement, &tail, T_identifier, suffix); } - hashmap_put(MACROS, macro->name, macro); + hashmap_put_borrowed(MACROS, macro->name, macro); } token_t *append_builtin_macro_token(token_t **replacement, @@ -199,18 +259,16 @@ void define_builtin_minimum_macro(const char *name, const char *maximum, const char *one) { - macro_t *macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); + const builtin_macro_token_t replacement[] = { + {T_open_bracket, "("}, {T_minus, "-"}, {T_numeric, maximum}, + {T_minus, "-"}, {T_numeric, one}, {T_close_bracket, ")"}, + }; + macro_t *macro = new_macro(intern_string((char *) name)); token_t *tail = NULL; - macro->name = intern_string((char *) name); - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, "("); - append_builtin_macro_token(¯o->replacement, &tail, T_minus, "-"); - append_builtin_macro_token(¯o->replacement, &tail, T_numeric, maximum); - append_builtin_macro_token(¯o->replacement, &tail, T_minus, "-"); - append_builtin_macro_token(¯o->replacement, &tail, T_numeric, one); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - hashmap_put(MACROS, macro->name, macro); + append_builtin_macro_tokens(¯o->replacement, &tail, replacement, + sizeof(replacement) / sizeof(*replacement)); + hashmap_put_borrowed(MACROS, macro->name, macro); } /* offsetof is specified as a function-like macro. Route its type/member @@ -219,64 +277,62 @@ void define_builtin_minimum_macro(const char *name, */ void install_stddef_offsetof_macro(void) { - macro_t *macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); + static const builtin_macro_token_t replacement[] = { + {T_identifier, "__builtin_offsetof"}, + {T_open_bracket, "("}, + {T_identifier, "type"}, + {T_comma, ","}, + {T_identifier, "member"}, + {T_close_bracket, ")"}, + }; + macro_t *macro = new_builtin_function_macro("offsetof", "type", "member"); token_t *tail = NULL; - macro->name = intern_string("offsetof"); - macro->is_function_like = true; - macro->param_num = 2; - macro->param_names[0] = new_token(T_identifier, &synth_built_in_loc, 4); - macro->param_names[0]->literal = "type"; - macro->param_names[1] = new_token(T_identifier, &synth_built_in_loc, 6); - macro->param_names[1]->literal = "member"; - - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "__builtin_offsetof"); - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, "("); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "type"); - append_builtin_macro_token(¯o->replacement, &tail, T_comma, ","); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "member"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - hashmap_put(MACROS, macro->name, macro); + append_builtin_macro_tokens(¯o->replacement, &tail, replacement, + sizeof(replacement) / sizeof(*replacement)); + hashmap_put_borrowed(MACROS, macro->name, macro); } /* shecc currently has signed plain char, 16-bit short, 32-bit int/long, and * 64-bit long long on every target. Keep tied to those actual * language types rather than to the host compiler's ABI. */ +#define LIMIT_OBJECT(name, value) \ + define_builtin_object_macro(name, T_numeric, value) +#define LIMIT_NEGATIVE(name, value) define_builtin_negative_macro(name, value) +#define LIMIT_MINIMUM(name, maximum, one) \ + define_builtin_minimum_macro(name, maximum, one) +#define LIMIT_CONSTRUCTION(name, suffix) \ + define_builtin_integer_construction_macro(name, suffix) + void install_limits_header(void) { - define_builtin_object_macro("CHAR_BIT", T_numeric, "8"); - define_builtin_object_macro("SCHAR_MAX", T_numeric, "127"); - define_builtin_object_macro("UCHAR_MAX", T_numeric, "255U"); - define_builtin_object_macro("CHAR_MAX", T_numeric, "127"); - define_builtin_object_macro("SHRT_MAX", T_numeric, "32767"); - define_builtin_object_macro("USHRT_MAX", T_numeric, "65535U"); - define_builtin_object_macro("INT_MAX", T_numeric, "2147483647"); - define_builtin_object_macro("UINT_MAX", T_numeric, "4294967295U"); - define_builtin_object_macro("LONG_MAX", T_numeric, "2147483647L"); - define_builtin_object_macro("ULONG_MAX", T_numeric, "4294967295UL"); - define_builtin_object_macro("LLONG_MAX", T_numeric, - "9223372036854775807LL"); - define_builtin_object_macro("ULLONG_MAX", T_numeric, - "18446744073709551615ULL"); - define_builtin_object_macro("MB_LEN_MAX", T_numeric, "1"); - - define_builtin_negative_macro("SCHAR_MIN", "128"); - define_builtin_negative_macro("CHAR_MIN", "128"); - define_builtin_negative_macro("SHRT_MIN", "32768"); - define_builtin_minimum_macro("INT_MIN", "2147483647", "1"); - define_builtin_minimum_macro("LONG_MIN", "2147483647L", "1L"); - define_builtin_minimum_macro("LLONG_MIN", "9223372036854775807LL", "1LL"); + LIMIT_OBJECT("CHAR_BIT", "8"); + LIMIT_OBJECT("SCHAR_MAX", "127"); + LIMIT_OBJECT("UCHAR_MAX", "255U"); + LIMIT_OBJECT("CHAR_MAX", "127"); + LIMIT_OBJECT("SHRT_MAX", "32767"); + LIMIT_OBJECT("USHRT_MAX", "65535U"); + LIMIT_OBJECT("INT_MAX", "2147483647"); + LIMIT_OBJECT("UINT_MAX", "4294967295U"); + LIMIT_OBJECT("LONG_MAX", "2147483647L"); + LIMIT_OBJECT("ULONG_MAX", "4294967295UL"); + LIMIT_OBJECT("LLONG_MAX", "9223372036854775807LL"); + LIMIT_OBJECT("ULLONG_MAX", "18446744073709551615ULL"); + LIMIT_OBJECT("MB_LEN_MAX", "1"); + + LIMIT_NEGATIVE("SCHAR_MIN", "128"); + LIMIT_NEGATIVE("CHAR_MIN", "128"); + LIMIT_NEGATIVE("SHRT_MIN", "32768"); + LIMIT_MINIMUM("INT_MIN", "2147483647", "1"); + LIMIT_MINIMUM("LONG_MIN", "2147483647L", "1L"); + LIMIT_MINIMUM("LLONG_MIN", "9223372036854775807LL", "1LL"); } /* A C null pointer constant may be the integer constant expression 0. */ void install_stddef_header(void) { - define_builtin_object_macro("NULL", T_numeric, "0"); + LIMIT_OBJECT("NULL", "0"); install_stddef_offsetof_macro(); } @@ -285,94 +341,83 @@ void install_stddef_header(void) */ void install_stdint_header(void) { - define_builtin_negative_macro("INT8_MIN", "128"); - define_builtin_object_macro("INT8_MAX", T_numeric, "127"); - define_builtin_object_macro("UINT8_MAX", T_numeric, "255U"); - define_builtin_negative_macro("INT16_MIN", "32768"); - define_builtin_object_macro("INT16_MAX", T_numeric, "32767"); - define_builtin_object_macro("UINT16_MAX", T_numeric, "65535U"); - define_builtin_minimum_macro("INT32_MIN", "2147483647", "1"); - define_builtin_object_macro("INT32_MAX", T_numeric, "2147483647"); - define_builtin_object_macro("UINT32_MAX", T_numeric, "4294967295U"); - define_builtin_minimum_macro("INT64_MIN", "9223372036854775807LL", "1LL"); - define_builtin_object_macro("INT64_MAX", T_numeric, - "9223372036854775807LL"); - define_builtin_object_macro("UINT64_MAX", T_numeric, - "18446744073709551615ULL"); - define_builtin_minimum_macro("INTMAX_MIN", "9223372036854775807LL", "1LL"); - define_builtin_object_macro("INTMAX_MAX", T_numeric, - "9223372036854775807LL"); - define_builtin_object_macro("UINTMAX_MAX", T_numeric, - "18446744073709551615ULL"); - define_builtin_minimum_macro("SIG_ATOMIC_MIN", "2147483647", "1"); - define_builtin_object_macro("SIG_ATOMIC_MAX", T_numeric, "2147483647"); - define_builtin_minimum_macro("WCHAR_MIN", "2147483647", "1"); - define_builtin_object_macro("WCHAR_MAX", T_numeric, "2147483647"); - define_builtin_object_macro("WINT_MIN", T_numeric, "0U"); - define_builtin_object_macro("WINT_MAX", T_numeric, "4294967295U"); - define_builtin_negative_macro("INT_LEAST8_MIN", "128"); - define_builtin_object_macro("INT_LEAST8_MAX", T_numeric, "127"); - define_builtin_object_macro("UINT_LEAST8_MAX", T_numeric, "255U"); - define_builtin_negative_macro("INT_LEAST16_MIN", "32768"); - define_builtin_object_macro("INT_LEAST16_MAX", T_numeric, "32767"); - define_builtin_object_macro("UINT_LEAST16_MAX", T_numeric, "65535U"); - define_builtin_minimum_macro("INT_LEAST32_MIN", "2147483647", "1"); - define_builtin_object_macro("INT_LEAST32_MAX", T_numeric, "2147483647"); - define_builtin_object_macro("UINT_LEAST32_MAX", T_numeric, "4294967295U"); - define_builtin_minimum_macro("INT_LEAST64_MIN", "9223372036854775807LL", - "1LL"); - define_builtin_object_macro("INT_LEAST64_MAX", T_numeric, - "9223372036854775807LL"); - define_builtin_object_macro("UINT_LEAST64_MAX", T_numeric, - "18446744073709551615ULL"); - define_builtin_minimum_macro("INT_FAST8_MIN", "2147483647", "1"); - define_builtin_object_macro("INT_FAST8_MAX", T_numeric, "2147483647"); - define_builtin_object_macro("UINT_FAST8_MAX", T_numeric, "4294967295U"); - define_builtin_minimum_macro("INT_FAST16_MIN", "2147483647", "1"); - define_builtin_object_macro("INT_FAST16_MAX", T_numeric, "2147483647"); - define_builtin_object_macro("UINT_FAST16_MAX", T_numeric, "4294967295U"); - define_builtin_minimum_macro("INT_FAST32_MIN", "2147483647", "1"); - define_builtin_object_macro("INT_FAST32_MAX", T_numeric, "2147483647"); - define_builtin_object_macro("UINT_FAST32_MAX", T_numeric, "4294967295U"); - define_builtin_minimum_macro("INT_FAST64_MIN", "9223372036854775807LL", - "1LL"); - define_builtin_object_macro("INT_FAST64_MAX", T_numeric, - "9223372036854775807LL"); - define_builtin_object_macro("UINT_FAST64_MAX", T_numeric, - "18446744073709551615ULL"); - define_builtin_integer_construction_macro("INT8_C", ""); - define_builtin_integer_construction_macro("UINT8_C", ""); - define_builtin_integer_construction_macro("INT16_C", ""); - define_builtin_integer_construction_macro("UINT16_C", ""); - define_builtin_integer_construction_macro("INT32_C", ""); - define_builtin_integer_construction_macro("UINT32_C", "U"); - define_builtin_integer_construction_macro("INT64_C", "LL"); - define_builtin_integer_construction_macro("UINT64_C", "ULL"); - define_builtin_integer_construction_macro("INTMAX_C", "LL"); - define_builtin_integer_construction_macro("UINTMAX_C", "ULL"); + LIMIT_NEGATIVE("INT8_MIN", "128"); + LIMIT_OBJECT("INT8_MAX", "127"); + LIMIT_OBJECT("UINT8_MAX", "255U"); + LIMIT_NEGATIVE("INT16_MIN", "32768"); + LIMIT_OBJECT("INT16_MAX", "32767"); + LIMIT_OBJECT("UINT16_MAX", "65535U"); + LIMIT_MINIMUM("INT32_MIN", "2147483647", "1"); + LIMIT_OBJECT("INT32_MAX", "2147483647"); + LIMIT_OBJECT("UINT32_MAX", "4294967295U"); + LIMIT_MINIMUM("INT64_MIN", "9223372036854775807LL", "1LL"); + LIMIT_OBJECT("INT64_MAX", "9223372036854775807LL"); + LIMIT_OBJECT("UINT64_MAX", "18446744073709551615ULL"); + LIMIT_MINIMUM("INTMAX_MIN", "9223372036854775807LL", "1LL"); + LIMIT_OBJECT("INTMAX_MAX", "9223372036854775807LL"); + LIMIT_OBJECT("UINTMAX_MAX", "18446744073709551615ULL"); + LIMIT_MINIMUM("SIG_ATOMIC_MIN", "2147483647", "1"); + LIMIT_OBJECT("SIG_ATOMIC_MAX", "2147483647"); + LIMIT_MINIMUM("WCHAR_MIN", "2147483647", "1"); + LIMIT_OBJECT("WCHAR_MAX", "2147483647"); + LIMIT_OBJECT("WINT_MIN", "0U"); + LIMIT_OBJECT("WINT_MAX", "4294967295U"); + LIMIT_NEGATIVE("INT_LEAST8_MIN", "128"); + LIMIT_OBJECT("INT_LEAST8_MAX", "127"); + LIMIT_OBJECT("UINT_LEAST8_MAX", "255U"); + LIMIT_NEGATIVE("INT_LEAST16_MIN", "32768"); + LIMIT_OBJECT("INT_LEAST16_MAX", "32767"); + LIMIT_OBJECT("UINT_LEAST16_MAX", "65535U"); + LIMIT_MINIMUM("INT_LEAST32_MIN", "2147483647", "1"); + LIMIT_OBJECT("INT_LEAST32_MAX", "2147483647"); + LIMIT_OBJECT("UINT_LEAST32_MAX", "4294967295U"); + LIMIT_MINIMUM("INT_LEAST64_MIN", "9223372036854775807LL", "1LL"); + LIMIT_OBJECT("INT_LEAST64_MAX", "9223372036854775807LL"); + LIMIT_OBJECT("UINT_LEAST64_MAX", "18446744073709551615ULL"); + LIMIT_MINIMUM("INT_FAST8_MIN", "2147483647", "1"); + LIMIT_OBJECT("INT_FAST8_MAX", "2147483647"); + LIMIT_OBJECT("UINT_FAST8_MAX", "4294967295U"); + LIMIT_MINIMUM("INT_FAST16_MIN", "2147483647", "1"); + LIMIT_OBJECT("INT_FAST16_MAX", "2147483647"); + LIMIT_OBJECT("UINT_FAST16_MAX", "4294967295U"); + LIMIT_MINIMUM("INT_FAST32_MIN", "2147483647", "1"); + LIMIT_OBJECT("INT_FAST32_MAX", "2147483647"); + LIMIT_OBJECT("UINT_FAST32_MAX", "4294967295U"); + LIMIT_MINIMUM("INT_FAST64_MIN", "9223372036854775807LL", "1LL"); + LIMIT_OBJECT("INT_FAST64_MAX", "9223372036854775807LL"); + LIMIT_OBJECT("UINT_FAST64_MAX", "18446744073709551615ULL"); + LIMIT_CONSTRUCTION("INT8_C", ""); + LIMIT_CONSTRUCTION("UINT8_C", ""); + LIMIT_CONSTRUCTION("INT16_C", ""); + LIMIT_CONSTRUCTION("UINT16_C", ""); + LIMIT_CONSTRUCTION("INT32_C", ""); + LIMIT_CONSTRUCTION("UINT32_C", "U"); + LIMIT_CONSTRUCTION("INT64_C", "LL"); + LIMIT_CONSTRUCTION("UINT64_C", "ULL"); + LIMIT_CONSTRUCTION("INTMAX_C", "LL"); + LIMIT_CONSTRUCTION("UINTMAX_C", "ULL"); if (PTR_SIZE == 8) { - define_builtin_minimum_macro("INTPTR_MIN", "9223372036854775807LL", - "1LL"); - define_builtin_object_macro("INTPTR_MAX", T_numeric, - "9223372036854775807LL"); - define_builtin_object_macro("UINTPTR_MAX", T_numeric, - "18446744073709551615ULL"); - define_builtin_minimum_macro("PTRDIFF_MIN", "9223372036854775807LL", - "1LL"); - define_builtin_object_macro("PTRDIFF_MAX", T_numeric, - "9223372036854775807LL"); - define_builtin_object_macro("SIZE_MAX", T_numeric, - "18446744073709551615ULL"); + LIMIT_MINIMUM("INTPTR_MIN", "9223372036854775807LL", "1LL"); + LIMIT_OBJECT("INTPTR_MAX", "9223372036854775807LL"); + LIMIT_OBJECT("UINTPTR_MAX", "18446744073709551615ULL"); + LIMIT_MINIMUM("PTRDIFF_MIN", "9223372036854775807LL", "1LL"); + LIMIT_OBJECT("PTRDIFF_MAX", "9223372036854775807LL"); + LIMIT_OBJECT("SIZE_MAX", "18446744073709551615ULL"); } else { - define_builtin_minimum_macro("INTPTR_MIN", "2147483647L", "1L"); - define_builtin_object_macro("INTPTR_MAX", T_numeric, "2147483647L"); - define_builtin_object_macro("UINTPTR_MAX", T_numeric, "4294967295UL"); - define_builtin_minimum_macro("PTRDIFF_MIN", "2147483647L", "1L"); - define_builtin_object_macro("PTRDIFF_MAX", T_numeric, "2147483647L"); - define_builtin_object_macro("SIZE_MAX", T_numeric, "4294967295UL"); + LIMIT_MINIMUM("INTPTR_MIN", "2147483647L", "1L"); + LIMIT_OBJECT("INTPTR_MAX", "2147483647L"); + LIMIT_OBJECT("UINTPTR_MAX", "4294967295UL"); + LIMIT_MINIMUM("PTRDIFF_MIN", "2147483647L", "1L"); + LIMIT_OBJECT("PTRDIFF_MAX", "2147483647L"); + LIMIT_OBJECT("SIZE_MAX", "4294967295UL"); } } +#undef LIMIT_OBJECT +#undef LIMIT_NEGATIVE +#undef LIMIT_MINIMUM +#undef LIMIT_CONSTRUCTION + /* stdbool.h is entirely macro-defined in C99. Supplying it here makes the * freestanding compiler usable with --no-libc too, rather than relying on the * private definitions prepended from lib/c.h. @@ -409,87 +454,63 @@ void install_iso646_header(void) */ void install_assert_header(void) { - macro_t *macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); + static const builtin_macro_token_t enabled[] = { + {T_open_bracket, "("}, + {T_open_bracket, "("}, + {T_identifier, "expr"}, + {T_close_bracket, ")"}, + {T_question, "?"}, + {T_open_bracket, "("}, + {T_identifier, "void"}, + {T_close_bracket, ")"}, + {T_numeric, "0"}, + {T_colon, ":"}, + {T_identifier, "__assert_fail"}, + {T_open_bracket, "("}, + {T_hash, "#"}, + {T_identifier, "expr"}, + {T_comma, ","}, + {T_identifier, "__FILE__"}, + {T_comma, ","}, + {T_identifier, "__LINE__"}, + {T_comma, ","}, + {T_identifier, "__func__"}, + {T_close_bracket, ")"}, + {T_close_bracket, ")"}, + }; + static const builtin_macro_token_t disabled[] = { + {T_open_bracket, "("}, + {T_identifier, "void"}, + {T_close_bracket, ")"}, + {T_numeric, "0"}, + }; + macro_t *macro = new_builtin_function_macro("assert", "expr", NULL); token_t *tail = NULL; - macro->name = intern_string("assert"); - macro->is_function_like = true; - macro->param_num = 1; - macro->param_names[0] = new_token(T_identifier, &synth_built_in_loc, 4); - macro->param_names[0]->literal = "expr"; - - if (!is_macro_defined("NDEBUG")) { - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, - "("); - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, - "("); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "expr"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - append_builtin_macro_token(¯o->replacement, &tail, T_question, "?"); - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, - "("); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "void"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - append_builtin_macro_token(¯o->replacement, &tail, T_numeric, "0"); - append_builtin_macro_token(¯o->replacement, &tail, T_colon, ":"); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "__assert_fail"); - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, - "("); - append_builtin_macro_token(¯o->replacement, &tail, T_hash, "#"); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "expr"); - append_builtin_macro_token(¯o->replacement, &tail, T_comma, ","); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "__FILE__"); - append_builtin_macro_token(¯o->replacement, &tail, T_comma, ","); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "__LINE__"); - append_builtin_macro_token(¯o->replacement, &tail, T_comma, ","); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "__func__"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - } else { + if (!is_macro_defined("NDEBUG")) + append_builtin_macro_tokens(¯o->replacement, &tail, enabled, + sizeof(enabled) / sizeof(*enabled)); + else { /* C99 requires the disabled form to remain a void expression while not * evaluating its operand. */ - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, - "("); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "void"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - append_builtin_macro_token(¯o->replacement, &tail, T_numeric, "0"); + append_builtin_macro_tokens(¯o->replacement, &tail, disabled, + sizeof(disabled) / sizeof(*disabled)); } - hashmap_put(MACROS, macro->name, macro); + hashmap_put_borrowed(MACROS, macro->name, macro); } -macro_t *new_stdarg_macro(const char *name, - const char *first_param, - const char *second_param) -{ - macro_t *macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - - macro->name = intern_string((char *) name); - macro->is_function_like = true; - macro->param_num = second_param ? 2 : 1; - macro->param_names[0] = - new_token(T_identifier, &synth_built_in_loc, strlen(first_param)); - macro->param_names[0]->literal = intern_string((char *) first_param); - if (second_param) { - macro->param_names[1] = - new_token(T_identifier, &synth_built_in_loc, strlen(second_param)); - macro->param_names[1]->literal = intern_string((char *) second_param); - } - return macro; -} +#define INSTALL_STDARG_MACRO(macro_name, first_param, second_param, \ + token_array) \ + do { \ + macro_t *macro = \ + new_builtin_function_macro(macro_name, first_param, second_param); \ + token_t *tail = NULL; \ + append_builtin_macro_tokens( \ + ¯o->replacement, &tail, token_array, \ + sizeof(token_array) / sizeof(*token_array)); \ + hashmap_put_borrowed(MACROS, macro->name, macro); \ + } while (0) /* The current ABI spills every variadic argument into a pointer-sized slot. * These macros deliberately cover integer and pointer arguments only; floating @@ -497,93 +518,40 @@ macro_t *new_stdarg_macro(const char *name, */ void install_stdarg_header(void) { - macro_t *macro; - token_t *tail; - - define_builtin_object_macro("__VA_SLOT_WORDS", T_numeric, - PTR_SIZE == 8 ? "2" : "1"); - define_builtin_object_macro("__VA_SLOT_BYTES", T_numeric, - PTR_SIZE == 8 ? "8" : "4"); - - macro = new_stdarg_macro("va_start", "ap", "last"); - tail = NULL; - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, "("); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, "ap"); - append_builtin_macro_token(¯o->replacement, &tail, T_assign, "="); - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, "("); - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, "("); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "va_list"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - append_builtin_macro_token(¯o->replacement, &tail, T_ampersand, "&"); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "last"); - append_builtin_macro_token(¯o->replacement, &tail, T_plus, "+"); - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, "("); - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, "("); - append_builtin_macro_token(¯o->replacement, &tail, T_sizeof, "sizeof"); - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, "("); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "last"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - append_builtin_macro_token(¯o->replacement, &tail, T_plus, "+"); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "__VA_SLOT_BYTES"); - append_builtin_macro_token(¯o->replacement, &tail, T_minus, "-"); - append_builtin_macro_token(¯o->replacement, &tail, T_numeric, "1"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - append_builtin_macro_token(¯o->replacement, &tail, T_divide, "/"); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "__VA_SLOT_BYTES"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - append_builtin_macro_token(¯o->replacement, &tail, T_asterisk, "*"); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "__VA_SLOT_WORDS"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - hashmap_put(MACROS, macro->name, macro); - - macro = new_stdarg_macro("va_arg", "ap", "type"); - tail = NULL; - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "__builtin_va_arg"); - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, "("); - append_builtin_macro_token(¯o->replacement, &tail, T_ampersand, "&"); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, "ap"); - append_builtin_macro_token(¯o->replacement, &tail, T_comma, ","); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "type"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - hashmap_put(MACROS, macro->name, macro); - - macro = new_stdarg_macro("va_copy", "dest", "src"); - tail = NULL; - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, "("); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "dest"); - append_builtin_macro_token(¯o->replacement, &tail, T_assign, "="); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, "src"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - hashmap_put(MACROS, macro->name, macro); - - macro = new_stdarg_macro("va_end", "ap", NULL); - tail = NULL; - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, "("); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "void"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - append_builtin_macro_token(¯o->replacement, &tail, T_numeric, "0"); - hashmap_put(MACROS, macro->name, macro); -} + static const builtin_macro_token_t va_start_replacement[] = { + {T_identifier, "__builtin_va_start"}, + {T_open_bracket, "("}, + {T_ampersand, "&"}, + {T_identifier, "ap"}, + {T_comma, ","}, + {T_identifier, "last"}, + {T_close_bracket, ")"}, + }; + static const builtin_macro_token_t va_arg_replacement[] = { + {T_identifier, "__builtin_va_arg"}, + {T_open_bracket, "("}, + {T_ampersand, "&"}, + {T_identifier, "ap"}, + {T_comma, ","}, + {T_identifier, "type"}, + {T_close_bracket, ")"}, + }; + static const builtin_macro_token_t va_copy_replacement[] = { + {T_open_bracket, "("}, {T_identifier, "dest"}, {T_assign, "="}, + {T_identifier, "src"}, {T_close_bracket, ")"}, + }; + static const builtin_macro_token_t va_end_replacement[] = { + {T_open_bracket, "("}, + {T_identifier, "void"}, + {T_close_bracket, ")"}, + {T_numeric, "0"}, + }; + INSTALL_STDARG_MACRO("va_start", "ap", "last", va_start_replacement); + INSTALL_STDARG_MACRO("va_arg", "ap", "type", va_arg_replacement); + INSTALL_STDARG_MACRO("va_copy", "dest", "src", va_copy_replacement); + INSTALL_STDARG_MACRO("va_end", "ap", NULL, va_end_replacement); +} +#undef INSTALL_STDARG_MACRO /* An angle header is lexed as ordinary preprocessing tokens. Recover its * spelling from the immutable source buffer so dotted and slash-separated names @@ -893,45 +861,6 @@ void pp_read_line_operands(token_t *directive, error_at("Unexpected token in #line directive", &cursor->location); } -int pp_get_operator_prio(opcode_t op) -{ - /* https://www.cs.uic.edu/~i109/Notes/COperatorPrecedenceTable.pdf */ - switch (op) { - case OP_ternary: - return 3; - case OP_log_or: - return 4; - case OP_log_and: - return 5; - case OP_bit_or: - return 6; - case OP_bit_xor: - return 7; - case OP_bit_and: - return 8; - case OP_eq: - case OP_neq: - return 9; - case OP_lt: - case OP_leq: - case OP_gt: - case OP_geq: - return 10; - case OP_lshift: - case OP_rshift: - return 11; - case OP_add: - case OP_sub: - return 12; - case OP_mul: - case OP_div: - case OP_mod: - return 13; - default: - return 0; - } -} - int pp_get_unary_operator_prio(opcode_t op) { switch (op) { @@ -957,75 +886,9 @@ token_t *pp_get_operator(token_t *tk, opcode_t *op, bool consume) error_at("Unexpected error when trying to evaulate constant operator", &tk->location); - switch (tk->next->kind) { - case T_plus: - op[0] = OP_add; - break; - case T_minus: - op[0] = OP_sub; - break; - case T_bit_not: - op[0] = OP_bit_not; - break; - case T_log_not: - op[0] = OP_log_not; - break; - case T_asterisk: - op[0] = OP_mul; - break; - case T_divide: - op[0] = OP_div; - break; - case T_mod: - op[0] = OP_mod; - break; - case T_lshift: - op[0] = OP_lshift; - break; - case T_rshift: - op[0] = OP_rshift; - break; - case T_log_and: - op[0] = OP_log_and; - break; - case T_log_or: - op[0] = OP_log_or; - break; - case T_eq: - op[0] = OP_eq; - break; - case T_noteq: - op[0] = OP_neq; - break; - case T_lt: - op[0] = OP_lt; - break; - case T_le: - op[0] = OP_leq; - break; - case T_gt: - op[0] = OP_gt; - break; - case T_ge: - op[0] = OP_geq; - break; - case T_ampersand: - op[0] = OP_bit_and; - break; - case T_bit_or: - op[0] = OP_bit_or; - break; - case T_bit_xor: - op[0] = OP_bit_xor; - break; - case T_question: - op[0] = OP_ternary; - break; - default: - /* Maybe it's an operand, we immediately return here. */ - op[0] = OP_generic; + op[0] = operator_for_token(tk->next->kind); + if (op[0] == OP_generic) return tk; - } return consume ? pp_lex_next_token(tk, true) : tk; } @@ -1184,22 +1047,6 @@ int pp_literal_will_overflow(const pp_integer_t *value, int base, int digit) digit > remainder); } -int pp_numeric_suffix_is_valid(const char *suffix) -{ - int pos = 0; - - if ((suffix[pos] | 32) == 'u') - pos++; - if ((suffix[pos] | 32) == 'l') { - pos++; - if ((suffix[pos] | 32) == 'l') - pos++; - } - if ((suffix[pos] | 32) == 'u') - pos++; - return suffix[pos] == '\0'; -} - void pp_multiply(pp_integer_t *lhs, const pp_integer_t *rhs) { pp_integer_t multiplicand, multiplier, product = {0}; @@ -1209,7 +1056,6 @@ void pp_multiply(pp_integer_t *lhs, const pp_integer_t *rhs) multiplicand.is_unsigned = lhs->is_unsigned; multiplier.lo = rhs->lo; multiplier.hi = rhs->hi; - multiplier.is_unsigned = rhs->is_unsigned; for (int i = 0; i < 64; i++) { if (multiplier.lo & 1) @@ -1297,19 +1143,14 @@ void pp_parse_integer_literal(token_t *tk, pp_integer_t *val) int is_decimal; pp_integer_t parsed = {0}; - if (literal[0] == '0') { - if ((literal[1] | 32) == 'x') { - base = 16; - i = 2; - } else if ((literal[1] | 32) == 'b') { - base = 2; - i = 2; - } else if (literal[1] && (literal[1] | 32) != 'u' && - (literal[1] | 32) != 'l') { - base = 8; - i = 1; - } - } + base = numeric_literal_base(literal); + if (base == 8 && literal[1] && + ((literal[1] | 32) == 'u' || (literal[1] | 32) == 'l')) + base = 10; + if (base == 16 || base == 2) + i = 2; + else if (base == 8) + i = 1; is_decimal = base == 10; while (literal[i] && (literal[i] | 32) != 'u' && (literal[i] | 32) != 'l') { int digit; @@ -1339,7 +1180,7 @@ void pp_parse_integer_literal(token_t *tk, pp_integer_t *val) error_at("Invalid integer constant suffix", &tk->location); i++; } - if (!pp_numeric_suffix_is_valid(suffix)) + if (!numeric_suffix_is_valid(suffix)) error_at("Invalid integer constant suffix", &tk->location); if (!val->is_unsigned && (parsed.hi & 0x80000000U)) { if (is_decimal) @@ -1371,6 +1212,30 @@ void pp_parse_integer_literal(token_t *tk, pp_integer_t *val) } } +static bool pp_compare(opcode_t op, + const pp_integer_t *lhs, + const pp_integer_t *rhs) +{ + int order = pp_compare_unsigned(lhs, rhs); + if (!lhs->is_unsigned && !rhs->is_unsigned && + ((lhs->hi ^ rhs->hi) & 0x80000000U)) + order = lhs->hi & 0x80000000U ? -1 : 1; + switch (op) { +#define RELATION(name, test) \ + case OP_##name: \ + return order test 0 + RELATION(eq, ==); + RELATION(neq, !=); + RELATION(lt, <); + RELATION(leq, <=); + RELATION(gt, >); + RELATION(geq, >=); +#undef RELATION + default: + return false; + } +} + token_t *pp_read_constant_infix_expr(int precedence, token_t *tk, pp_integer_t *val, @@ -1581,13 +1446,12 @@ token_t *pp_read_constant_expr_operand(token_t *tk, return tk; } -token_t *pp_read_constant_infix_expr(int precedence, - token_t *tk, - pp_integer_t *val, - bool evaluate) +static token_t *pp_read_constant_infix_body(int precedence, + token_t *tk, + pp_integer_t *val, + bool evaluate) { pp_integer_t lhs = {0}, rhs = {0}; - int comparison; /* Evaluate unary expression first */ opcode_t op; @@ -1626,7 +1490,7 @@ token_t *pp_read_constant_infix_expr(int precedence, while (true) { tk = pp_get_operator(tk, &op, false); - current_precedence = pp_get_operator_prio(op); + current_precedence = get_operator_prio(op); if (current_precedence == 0 || current_precedence <= precedence) break; @@ -1670,14 +1534,7 @@ token_t *pp_read_constant_infix_expr(int precedence, pp_enum_usual_arithmetic(&lhs, &rhs); switch (op) { - case OP_add: - case OP_sub: - case OP_mul: - case OP_div: - case OP_mod: - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: + OP_USUAL_ARITHMETIC_CASES: lhs.is_unsigned = PP_USES_UNSIGNED(&lhs, &rhs); break; default: @@ -1750,46 +1607,12 @@ token_t *pp_read_constant_infix_expr(int precedence, pp_shift_right_one(&lhs, !lhs.is_unsigned); break; case OP_gt: - if (!lhs.is_unsigned && !rhs.is_unsigned && - (lhs.hi >> 31) != (rhs.hi >> 31)) - pp_set_boolean(&lhs, !(lhs.hi >> 31)); - else { - comparison = pp_compare_unsigned(&lhs, &rhs); - pp_set_boolean(&lhs, comparison > 0); - } - break; case OP_geq: - if (!lhs.is_unsigned && !rhs.is_unsigned && - (lhs.hi >> 31) != (rhs.hi >> 31)) - pp_set_boolean(&lhs, !(lhs.hi >> 31)); - else { - comparison = pp_compare_unsigned(&lhs, &rhs); - pp_set_boolean(&lhs, comparison >= 0); - } - break; case OP_lt: - if (!lhs.is_unsigned && !rhs.is_unsigned && - (lhs.hi >> 31) != (rhs.hi >> 31)) - pp_set_boolean(&lhs, lhs.hi >> 31); - else { - comparison = pp_compare_unsigned(&lhs, &rhs); - pp_set_boolean(&lhs, comparison < 0); - } - break; case OP_leq: - if (!lhs.is_unsigned && !rhs.is_unsigned && - (lhs.hi >> 31) != (rhs.hi >> 31)) - pp_set_boolean(&lhs, lhs.hi >> 31); - else { - comparison = pp_compare_unsigned(&lhs, &rhs); - pp_set_boolean(&lhs, comparison <= 0); - } - break; case OP_eq: - pp_set_boolean(&lhs, lhs.lo == rhs.lo && lhs.hi == rhs.hi); - break; case OP_neq: - pp_set_boolean(&lhs, lhs.lo != rhs.lo || lhs.hi != rhs.hi); + pp_set_boolean(&lhs, pp_compare(op, &lhs, &rhs)); break; case OP_log_and: pp_set_boolean(&lhs, pp_is_true(&lhs) && pp_is_true(&rhs)); @@ -1815,6 +1638,25 @@ token_t *pp_read_constant_infix_expr(int precedence, return tk; } +/* Each parenthesis, unary operator and conditional of a #if expression recurses + * once here, so `#if ((((...1...))))' nested tens of thousands deep exhausted + * the machine stack. Bound it with the limit the C expression parser uses. + */ +static int pp_expr_depth = 0; + +token_t *pp_read_constant_infix_expr(int precedence, + token_t *tk, + pp_integer_t *val, + bool evaluate) +{ + pp_expr_depth++; + if (pp_expr_depth > MAX_EXPR_DEPTH) + error_at("Expression nesting too deep", tk ? &tk->location : NULL); + tk = pp_read_constant_infix_body(precedence, tk, val, evaluate); + pp_expr_depth--; + return tk; +} + token_t *pp_read_constant_expr(token_t *tk, pp_integer_t *val) { tk = pp_read_constant_infix_expr(0, tk, val, true); @@ -1897,7 +1739,7 @@ token_t *pp_stringify(token_t *arg, source_location_t *loc) text[n] = 0; token_t *out = new_token(T_string, loc, n); - out->literal = arena_strdup(TOKEN_ARENA, text); + out->literal = intern_string(text); return out; } @@ -1962,7 +1804,8 @@ token_t *pp_pragma_operator(token_t *tk, token_t *owner) while (len > 0 && (p[len - 1] == ' ' || p[len - 1] == '\t')) len--; if (len == 4 && !strncmp(p, "once", 4)) - hashmap_put(PRAGMA_ONCE, owner->location.physical_filename, NULL); + hashmap_put_borrowed(PRAGMA_ONCE, owner->location.physical_filename, + NULL); return pp_lex_expect_token(tk, T_close_bracket, true); } @@ -2027,6 +1870,23 @@ token_t *pp_paste_tokens(token_t *lhs, token_t *rhs, source_location_t *loc) * own. Membership in @args, rather than a non-empty value, is what makes a name * a parameter. */ +static bool pp_append_significant_tokens(token_t *tokens, + token_t **tail, + token_t **tail_prev) +{ + bool appended = false; + + for (token_t *tk = tokens; tk; tk = tk->next) { + if (pp_is_layout(tk)) + continue; + *tail_prev = *tail; + (*tail)->next = copy_token(tk); + *tail = (*tail)->next; + appended = true; + } + return appended; +} + token_t *pp_subst_hash(token_t *rep, hashmap_t *args) { token_t head; @@ -2073,17 +1933,8 @@ token_t *pp_subst_hash(token_t *rep, hashmap_t *args) !strcmp(operand->literal, "__VA_ARGS__") && hashmap_contains(args, operand->literal)) { token_t *comma_prev = tail_prev; - bool any = false; - - for (token_t *t = hashmap_get(args, operand->literal); t; - t = t->next) { - if (pp_is_layout(t)) - continue; - tail_prev = tail; - tail->next = copy_token(t); - tail = tail->next; - any = true; - } + bool any = pp_append_significant_tokens( + hashmap_get(args, operand->literal), &tail, &tail_prev); if (!any) { tail = comma_prev; tail->next = NULL; @@ -2130,13 +1981,7 @@ token_t *pp_subst_hash(token_t *rep, hashmap_t *args) * emit the remainder of the macro body twice. */ if (rhs_is_arg) { - for (token_t *rest = rhs->next; rest; rest = rest->next) { - if (pp_is_layout(rest)) - continue; - tail_prev = tail; - tail->next = copy_token(rest); - tail = tail->next; - } + pp_append_significant_tokens(rhs->next, &tail, &tail_prev); } lhs_empty = false; } @@ -2152,16 +1997,7 @@ token_t *pp_subst_hash(token_t *rep, hashmap_t *args) if (args && tk->kind == T_identifier && after && after->kind == T_hashhash && hashmap_contains(args, tk->literal)) { token_t *arg = hashmap_get(args, tk->literal); - bool any = false; - - for (token_t *t = arg; t; t = t->next) { - if (pp_is_layout(t)) - continue; - tail_prev = tail; - tail->next = copy_token(t); - tail = tail->next; - any = true; - } + bool any = pp_append_significant_tokens(arg, &tail, &tail_prev); lhs_present = true; lhs_empty = !any; continue; @@ -2183,7 +2019,82 @@ token_t *pp_subst_hash(token_t *rep, hashmap_t *args) return head.next; } -token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) +/* An expansion whose last token names a function-like macro calls it when the + * stream being rescanned follows with '(': a parameter bound to ADD in a body + * "f(a, b)", "#define F ADD" before "F(1, 2)", or "ID(ADD)(1, 2)". The standard + * rescans the expansion together with the rest of the stream; this expander + * rescans the expansion alone, so hand that name back. Splice what precedes it + * in @expanded onto *@cur and return a copy of it chained to the stream after + * @tk, or NULL when no such call follows. + * + * @hidden is the hide set the name came out of: a macro's own name in its + * expansion is never invoked again, so "#define f(x) f" leaves "f(1)(2)" as + * "f(2)", as gcc does. Each resumption consumes a '(' from the stream, so it + * terminates. + */ +static token_t *pp_resume_callee(token_t *expanded, + token_t *end, + token_t *tk, + const preprocess_ctx_t *ctx, + hide_set_t *hidden, + token_t **cur) +{ + token_t *callee = NULL; + token_t *before = NULL; + token_t *prev = NULL; + token_t *resume; + macro_t *macro; + + if (!expanded || !pp_lex_peek_token(tk, T_open_bracket, true)) + return NULL; + for (token_t *t = expanded; t; t = t->next) { + if (!pp_is_layout(t)) { + callee = t; + before = prev; + } + if (t == end) + break; + prev = t; + } + /* A parameter's own name would be substituted again, not invoked. */ + if (!callee || callee->kind != T_identifier || + (ctx->macro_args && + hashmap_contains(ctx->macro_args, callee->literal)) || + hide_set_contains(hidden, callee->literal)) + return NULL; + macro = hashmap_get(MACROS, callee->literal); + if (!macro || !macro->is_function_like || macro->is_disabled) + return NULL; + if (before) { + (*cur)->next = expanded; + *cur = before; + } + resume = copy_token(callee); + resume->next = tk->next; + return resume; +} + +static bool pp_resume_or_append(token_t *expanded, + token_t *end, + token_t **tk, + const preprocess_ctx_t *ctx, + hide_set_t *hidden, + token_t **cur) +{ + token_t *resume = pp_resume_callee(expanded, end, *tk, ctx, hidden, cur); + + if (resume) { + *tk = resume; + return true; + } + if (expanded) { + (*cur)->next = expanded; + *cur = end; + } + return false; +} + +static token_t *pp_preprocess_body(token_t *tk, preprocess_ctx_t *ctx) { token_t head; token_t *cur = &head; @@ -2245,10 +2156,11 @@ token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) */ macro_arg_replacement = pp_preprocess_internal( macro_arg_replacement, &expansion_ctx); - if (macro_arg_replacement) { - cur->next = macro_arg_replacement; - cur = expansion_ctx.end_of_token; - } + + if (pp_resume_or_append(macro_arg_replacement, + expansion_ctx.end_of_token, &tk, + ctx, ctx->hide_set, &cur)) + continue; } tk = pp_lex_next_token(tk, false); continue; @@ -2374,13 +2286,14 @@ token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) */ if (empty_zero_arg_call) { if (macro->is_variadic) - hashmap_put(expansion_ctx.macro_args, - macro->variadic_tk->literal, NULL); + hashmap_put_borrowed(expansion_ctx.macro_args, + macro->variadic_tk->literal, + NULL); /* Bind argument to corresponding parameter */ } else if (arg_idx < macro->param_num) { param_tk = macro->param_names[arg_idx++]; - hashmap_put(expansion_ctx.macro_args, param_tk->literal, - arg_head.next); + hashmap_put_borrowed(expansion_ctx.macro_args, + param_tk->literal, arg_head.next); } else { /* Handle variadic macro overflow * @@ -2405,10 +2318,10 @@ token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) prev->next = new_token(T_comma, ¶m_tk->location, 1); prev->next->next = arg_head.next; - prev = arg_cur; } else { - hashmap_put(expansion_ctx.macro_args, - param_tk->literal, arg_head.next); + hashmap_put_borrowed(expansion_ctx.macro_args, + param_tk->literal, + arg_head.next); } } else { error_at( @@ -2445,8 +2358,8 @@ token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) if (macro->is_variadic && !hashmap_contains(expansion_ctx.macro_args, macro->variadic_tk->literal)) - hashmap_put(expansion_ctx.macro_args, - macro->variadic_tk->literal, NULL); + hashmap_put_borrowed(expansion_ctx.macro_args, + macro->variadic_tk->literal, NULL); /* Expand macro body with collected arguments Replace parameter * references with supplied argument tokens @@ -2454,12 +2367,17 @@ token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) token_t *expanded = pp_preprocess_internal( pp_subst_hash(macro->replacement, expansion_ctx.macro_args), &expansion_ctx); - if (expanded) { - cur->next = expanded; - cur = expansion_ctx.end_of_token; - } - hashmap_free(expansion_ctx.macro_args); + if (pp_resume_or_append(expanded, expansion_ctx.end_of_token, + &tk, ctx, expansion_ctx.hide_set, &cur)) + continue; + } else if (macro->is_function_like) { + /* A function-like macro expands only when followed by '('. Its + * bare identifier may be an ordinary identifier or a later + * macro argument, so preserve it unchanged. + */ + cur->next = copy_token(tk); + cur = cur->next; } else { /* Handle object-like macro expansion (no parameters) Simply * expand the replacement with current hide set plus this macro @@ -2469,10 +2387,9 @@ token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) hide_set_union(ctx->hide_set, new_hide_set(tk->literal)); token_t *expanded = pp_preprocess_internal( pp_subst_hash(macro->replacement, NULL), &expansion_ctx); - if (expanded) { - cur->next = expanded; - cur = expansion_ctx.end_of_token; - } + if (pp_resume_or_append(expanded, expansion_ctx.end_of_token, + &tk, ctx, expansion_ctx.hide_set, &cur)) + continue; } tk = pp_lex_next_token(tk, false); @@ -2602,6 +2519,14 @@ token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) inclusion_path[MAX_LINE_LEN - 1] = '\0'; } + /* A header that is missing is the directive's mistake, and is + * reported there. Reading it here also leaves it cached for the + * lexer, under the name the lexer will ask for. + */ + if (!try_get_file_buf(normalize_filename(inclusion_path))) + error_at("Included file cannot be found", + &include_tk->location); + tk = pp_lex_expect_token(tk, T_newline, true); tk = pp_lex_next_token(tk, false); @@ -2632,8 +2557,7 @@ token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) macro = hashmap_get(MACROS, tk->literal); if (!macro) { - macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - macro->name = tk->literal; + macro = new_macro(tk->literal); } else { /* Ensures that #undef effect is overwritten */ macro->is_disabled = false; @@ -2712,7 +2636,7 @@ token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) tk = pp_lex_expect_token(tk, T_newline, false); tk = pp_lex_next_token(tk, false); macro->replacement = r_head; - hashmap_put(MACROS, macro->name, macro); + hashmap_put_borrowed(MACROS, macro->name, macro); continue; } case T_cppd_undef: { @@ -2795,8 +2719,8 @@ token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) tk = pp_lex_next_token(tk, true); if (!strcmp("once", tk->literal)) - hashmap_put(PRAGMA_ONCE, tk->location.physical_filename, - NULL); + hashmap_put_borrowed(PRAGMA_ONCE, + tk->location.physical_filename, NULL); } while (!pp_lex_peek_token(tk, T_newline, true)) @@ -2834,7 +2758,6 @@ token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) * included previously while created by #define. */ error_at("Backslash is not allowed here", &cur->location); - break; } case T_eof: { if (ctx->trim_eof) { @@ -2863,6 +2786,24 @@ token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) return head.next; } +/* Each nested macro argument, rescanned replacement and included file re-enters + * the expander, whose frame is large, so `F(F(F(...)))' nested a few thousand + * deep exhausted the machine stack. Bound the recursion and report it instead. + */ +static int pp_depth = 0; + +token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) +{ + token_t *result; + + pp_depth++; + if (pp_depth > MAX_MACRO_DEPTH) + error_at("Macro expansion nesting too deep", tk ? &tk->location : NULL); + result = pp_preprocess_body(tk, ctx); + pp_depth--; + return result; +} + /* Report whether a string literal payload ends inside a numeric escape that a * following digit would extend: a hexadecimal escape, whose digit run has no * limit, or an octal escape of fewer than three digits. @@ -3001,72 +2942,62 @@ token_t *preprocess(token_t *tk) synth_built_in_loc.line = 1; synth_built_in_loc.filename = ""; - macro_t *macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - macro->name = "__FILE__"; + macro_t *macro = new_macro("__FILE__"); macro->handler = file_macro_handler; - hashmap_put(MACROS, "__FILE__", macro); + hashmap_put_borrowed(MACROS, "__FILE__", macro); - macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - macro->name = "__LINE__"; + macro = new_macro("__LINE__"); macro->handler = line_macro_handler; - hashmap_put(MACROS, "__LINE__", macro); + hashmap_put_borrowed(MACROS, "__LINE__", macro); - macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - macro->name = "__DATE__"; + macro = new_macro("__DATE__"); macro->handler = date_macro_handler; - hashmap_put(MACROS, "__DATE__", macro); + hashmap_put_borrowed(MACROS, "__DATE__", macro); - macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - macro->name = "__TIME__"; + macro = new_macro("__TIME__"); macro->handler = time_macro_handler; - hashmap_put(MACROS, "__TIME__", macro); + hashmap_put_borrowed(MACROS, "__TIME__", macro); /* C99-required implementation macros. shecc supplies its own small runtime * rather than a complete hosted library, so advertise freestanding mode * while retaining the C99 language-version identifier. */ - macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - macro->name = "__STDC__"; + macro = new_macro("__STDC__"); macro->replacement = new_token(T_numeric, &synth_built_in_loc, 1); macro->replacement->literal = "1"; - hashmap_put(MACROS, "__STDC__", macro); + hashmap_put_borrowed(MACROS, "__STDC__", macro); - macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - macro->name = "__STDC_VERSION__"; + macro = new_macro("__STDC_VERSION__"); macro->replacement = new_token(T_numeric, &synth_built_in_loc, 7); macro->replacement->literal = "199901L"; - hashmap_put(MACROS, "__STDC_VERSION__", macro); + hashmap_put_borrowed(MACROS, "__STDC_VERSION__", macro); - macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - macro->name = "__STDC_HOSTED__"; + macro = new_macro("__STDC_HOSTED__"); macro->replacement = new_token(T_numeric, &synth_built_in_loc, 1); macro->replacement->literal = "0"; - hashmap_put(MACROS, "__STDC_HOSTED__", macro); + hashmap_put_borrowed(MACROS, "__STDC_HOSTED__", macro); /* architecture defines */ - macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - macro->name = ARCH_PREDEFINED; + macro = new_macro(ARCH_PREDEFINED); macro->replacement = new_token(T_numeric, &synth_built_in_loc, 1); macro->replacement->literal = "1"; - hashmap_put(MACROS, ARCH_PREDEFINED, macro); + hashmap_put_borrowed(MACROS, ARCH_PREDEFINED, macro); /* shecc run-time defines */ - macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - macro->name = "__SHECC__"; + macro = new_macro("__SHECC__"); macro->replacement = new_token(T_numeric, &synth_built_in_loc, 1); macro->replacement->literal = "1"; - hashmap_put(MACROS, "__SHECC__", macro); + hashmap_put_borrowed(MACROS, "__SHECC__", macro); /* Tells the source being compiled that the embedded libc is not part of the * output, so the functions lib/c.c would have supplied -- '__syscall' above * all -- are unavailable and libc resolves through the PLT instead. */ if (dynlink) { - macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - macro->name = "__SHECC_DYNLINK__"; + macro = new_macro("__SHECC_DYNLINK__"); macro->replacement = new_token(T_numeric, &synth_built_in_loc, 1); macro->replacement->literal = "1"; - hashmap_put(MACROS, "__SHECC_DYNLINK__", macro); + hashmap_put_borrowed(MACROS, "__SHECC_DYNLINK__", macro); } tk = pp_preprocess_internal(tk, &ctx); @@ -3076,6 +3007,10 @@ token_t *preprocess(token_t *tk) return tk; } +#define TOKEN_SPELLING(kind, spelling) \ + case kind: \ + return spelling + char *token_to_string(token_t *tk, char *dest) { switch (tk->kind) { @@ -3103,164 +3038,85 @@ char *token_to_string(token_t *tk, char *dest) case T_wchar: snprintf(dest, MAX_TOKEN_LEN, "L'%s'", tk->literal); return dest; - case T_comma: - return ","; - case T_open_bracket: - return "("; - case T_close_bracket: - return ")"; - case T_open_curly: - return "{"; - case T_close_curly: - return "}"; - case T_open_square: - return "["; - case T_close_square: - return "]"; - case T_asterisk: - return "*"; - case T_divide: - return "/"; - case T_mod: - return "%"; - case T_bit_or: - return "|"; - case T_bit_xor: - return "^"; - case T_bit_not: - return "~"; - case T_log_and: - return "&&"; - case T_log_or: - return "||"; - case T_log_not: - return "!"; - case T_lt: - return "<"; - case T_gt: - return ">"; - case T_le: - return "<="; - case T_ge: - return ">="; - case T_lshift: - return "<<"; - case T_rshift: - return ">>"; - case T_dot: - return "."; - case T_arrow: - return "->"; - case T_plus: - return "+"; - case T_minus: - return "-"; - case T_minuseq: - return "-="; - case T_pluseq: - return "+="; - case T_asteriskeq: - return "*="; - case T_divideeq: - return "/="; - case T_modeq: - return "%="; - case T_lshifteq: - return "<<="; - case T_rshifteq: - return ">>="; - case T_xoreq: - return "^="; - case T_oreq: - return "|="; - case T_andeq: - return "&="; - case T_eq: - return "=="; - case T_noteq: - return "!="; - case T_assign: - return "="; - case T_increment: - return "++"; - case T_decrement: - return "--"; - case T_question: - return "?"; - case T_colon: - return ":"; - case T_semicolon: - return ";"; - case T_ampersand: - return "&"; - case T_return: - return "return"; - case T_if: - return "if"; - case T_else: - return "else"; - case T_while: - return "while"; - case T_for: - return "for"; - case T_do: - return "do"; - case T_typedef: - return "typedef"; - case T_enum: - return "enum"; - case T_struct: - return "struct"; - case T_union: - return "union"; - case T_sizeof: - return "sizeof"; - case T_elipsis: - return "..."; - case T_switch: - return "switch"; - case T_case: - return "case"; - case T_break: - return "break"; - case T_default: - return "default"; - case T_continue: - return "continue"; - case T_goto: - return "goto"; - case T_const: - return "const"; - case T_volatile: - return "volatile"; - case T_static: - return "static"; - case T_extern: - return "extern"; - case T_register: - return "register"; - case T_auto: - return "auto"; - case T_restrict: - return "restrict"; - case T_inline: - return "inline"; - case T_signed: - return "signed"; - case T_unsigned: - return "unsigned"; - case T_long: - return "long"; - case T_float: - return "float"; - case T_double: - return "double"; - case T_complex: - return "_Complex"; - case T_imaginary: - return "_Imaginary"; - case T_newline: - return "\n"; + TOKEN_SPELLING(T_comma, ","); + TOKEN_SPELLING(T_open_bracket, "("); + TOKEN_SPELLING(T_close_bracket, ")"); + TOKEN_SPELLING(T_open_curly, "{"); + TOKEN_SPELLING(T_close_curly, "}"); + TOKEN_SPELLING(T_open_square, "["); + TOKEN_SPELLING(T_close_square, "]"); + TOKEN_SPELLING(T_asterisk, "*"); + TOKEN_SPELLING(T_divide, "/"); + TOKEN_SPELLING(T_mod, "%"); + TOKEN_SPELLING(T_bit_or, "|"); + TOKEN_SPELLING(T_bit_xor, "^"); + TOKEN_SPELLING(T_bit_not, "~"); + TOKEN_SPELLING(T_log_and, "&&"); + TOKEN_SPELLING(T_log_or, "||"); + TOKEN_SPELLING(T_log_not, "!"); + TOKEN_SPELLING(T_lt, "<"); + TOKEN_SPELLING(T_gt, ">"); + TOKEN_SPELLING(T_le, "<="); + TOKEN_SPELLING(T_ge, ">="); + TOKEN_SPELLING(T_lshift, "<<"); + TOKEN_SPELLING(T_rshift, ">>"); + TOKEN_SPELLING(T_dot, "."); + TOKEN_SPELLING(T_arrow, "->"); + TOKEN_SPELLING(T_plus, "+"); + TOKEN_SPELLING(T_minus, "-"); + TOKEN_SPELLING(T_minuseq, "-="); + TOKEN_SPELLING(T_pluseq, "+="); + TOKEN_SPELLING(T_asteriskeq, "*="); + TOKEN_SPELLING(T_divideeq, "/="); + TOKEN_SPELLING(T_modeq, "%="); + TOKEN_SPELLING(T_lshifteq, "<<="); + TOKEN_SPELLING(T_rshifteq, ">>="); + TOKEN_SPELLING(T_xoreq, "^="); + TOKEN_SPELLING(T_oreq, "|="); + TOKEN_SPELLING(T_andeq, "&="); + TOKEN_SPELLING(T_eq, "=="); + TOKEN_SPELLING(T_noteq, "!="); + TOKEN_SPELLING(T_assign, "="); + TOKEN_SPELLING(T_increment, "++"); + TOKEN_SPELLING(T_decrement, "--"); + TOKEN_SPELLING(T_question, "?"); + TOKEN_SPELLING(T_colon, ":"); + TOKEN_SPELLING(T_semicolon, ";"); + TOKEN_SPELLING(T_ampersand, "&"); + TOKEN_SPELLING(T_return, "return"); + TOKEN_SPELLING(T_if, "if"); + TOKEN_SPELLING(T_else, "else"); + TOKEN_SPELLING(T_while, "while"); + TOKEN_SPELLING(T_for, "for"); + TOKEN_SPELLING(T_do, "do"); + TOKEN_SPELLING(T_typedef, "typedef"); + TOKEN_SPELLING(T_enum, "enum"); + TOKEN_SPELLING(T_struct, "struct"); + TOKEN_SPELLING(T_union, "union"); + TOKEN_SPELLING(T_sizeof, "sizeof"); + TOKEN_SPELLING(T_elipsis, "..."); + TOKEN_SPELLING(T_switch, "switch"); + TOKEN_SPELLING(T_case, "case"); + TOKEN_SPELLING(T_break, "break"); + TOKEN_SPELLING(T_default, "default"); + TOKEN_SPELLING(T_continue, "continue"); + TOKEN_SPELLING(T_goto, "goto"); + TOKEN_SPELLING(T_const, "const"); + TOKEN_SPELLING(T_volatile, "volatile"); + TOKEN_SPELLING(T_static, "static"); + TOKEN_SPELLING(T_extern, "extern"); + TOKEN_SPELLING(T_register, "register"); + TOKEN_SPELLING(T_auto, "auto"); + TOKEN_SPELLING(T_restrict, "restrict"); + TOKEN_SPELLING(T_inline, "inline"); + TOKEN_SPELLING(T_signed, "signed"); + TOKEN_SPELLING(T_unsigned, "unsigned"); + TOKEN_SPELLING(T_long, "long"); + TOKEN_SPELLING(T_float, "float"); + TOKEN_SPELLING(T_double, "double"); + TOKEN_SPELLING(T_complex, "_Complex"); + TOKEN_SPELLING(T_imaginary, "_Imaginary"); + TOKEN_SPELLING(T_newline, "\n"); case T_backslash: error_at( "Internal error, backslash should be ommited after " @@ -3276,6 +3132,8 @@ char *token_to_string(token_t *tk, char *dest) } case T_tab: return "\t"; + case T_translation_unit: + return "\n"; case T_start: /* FIXME: Unused token kind */ break; @@ -3305,16 +3163,22 @@ char *token_to_string(token_t *tk, char *dest) return NULL; } +#undef TOKEN_SPELLING + void emit_preprocessed_token(token_t *tk) { char token_buffer[MAX_TOKEN_LEN], *literal; + token_kind_t previous_kind = T_start; while (tk) { - literal = token_to_string(tk, token_buffer); + literal = tk->kind == T_translation_unit && previous_kind == T_newline + ? "" + : token_to_string(tk, token_buffer); if (literal) printf("%s", literal); + previous_kind = tk->kind; tk = tk->next; } } diff --git a/src/reg-alloc.c b/src/reg-alloc.c deleted file mode 100644 index 6960043e..00000000 --- a/src/reg-alloc.c +++ /dev/null @@ -1,4143 +0,0 @@ -/* - * shecc - Self-Hosting and Educational C Compiler. - * - * shecc is freely redistributable under the BSD 2 clause license. See the file - * "LICENSE" for information on usage and redistribution of this file. - */ - -/* Allocate registers from IR. The linear-scan algorithm now expects a minimum - * of 7 available registers (typical for RISC-style architectures). - * - * TODO: Implement "-O level" optimization control. Currently the allocator - * always performs dead variable elimination without writing back to stack. - */ -#include "defs.h" -#include "globals.c" - -/* A value is pointer-like if it is a pointer itself or its type is. On LP64 - * targets these occupy PTR_SIZE bytes rather than 4. - */ -bool is_pointer_like(var_t *v) -{ - return v && (v->ptr_level > 0 || (v->type && v->type->ptr_level > 0)); -} - -/* An operand holds an address if it is pointer-like or is an array, which - * decays to one wherever it is used as a value. A subscript reaches OP_add with - * the array itself as the base, so leaving arrays out here would let an LP64 - * backend read a[i] as ordinary arithmetic and skip widening the index. - */ -bool is_address_like(var_t *v) -{ - return is_pointer_like(v) || (v && v->array_size > 0); -} - -/* Record which operands of a three-address instruction are addresses. - * - * An LP64 backend needs the operands apart, not just the instruction: pointer - * arithmetic keeps the address 64 bits wide while sign-extending the int index - * beside it. - * - * The two source flags and is_pointer deliberately ask different questions. - * Only the AArch64 backend reads the source flags, and it wants every address, - * arrays included. is_pointer is older and the x86-64 backend decides store - * widths and int narrowing by it, so it keeps counting pointer-like operands - * alone rather than acquiring arrays and changing a settled target. - */ -bool is_unsigned_scalar(const var_t *var) -{ - return var && !var->ptr_level && !var->is_func && var->type && - var->type->is_unsigned; -} - -void set_ptr_flags(ph2_ir_t *ir, insn_t *insn) -{ - ir->src0_is_pointer = is_address_like(insn->rs1); - ir->src1_is_pointer = is_address_like(insn->rs2); - ir->is_pointer = is_pointer_like(insn->rd) || is_pointer_like(insn->rs1) || - is_pointer_like(insn->rs2); - ir->is_unsigned = is_unsigned_scalar(insn->rd) || - is_unsigned_scalar(insn->rs1) || - is_unsigned_scalar(insn->rs2); - ir->src0_is_unsigned = is_unsigned_scalar(insn->rs1); - ir->src1_is_unsigned = is_unsigned_scalar(insn->rs2); - - /* Addresses are ordered as unsigned values (C99 6.5.8): one above the - * signed range still compares greater than one below it. - */ - if ((insn->opcode == OP_lt || insn->opcode == OP_leq || - insn->opcode == OP_gt || insn->opcode == OP_geq) && - (ir->src0_is_pointer || ir->src1_is_pointer)) - ir->src0_is_unsigned = true; -} - -/* Width of the value a local's frame slot actually holds. - * - * Slots are pointer-sized, but a scalar occupies only its low bytes. Reading - * the whole slot back would pick up stale bytes whenever the variable was - * written through a pointer, so loads must use the declared width. Anything not - * a known scalar -- pointers, arrays, aggregates, functions -- keeps a full - * pointer in its slot and is reported as such. - */ -int var_slot_size(var_t *v) -{ - if (!v || v->ptr_level || v->is_func || v->array_size) - return PTR_SIZE; - - /* Only a variable whose address escaped can have its slot written behind - * the allocator's back, at the pointee's width rather than the slot's. - * Everything else is written and read through this same path, so the full - * slot is always valid and the wider access is safe. - * - * Narrowing these to the declared width does not work: an SSA temporary - * carries the type of the expression that made it, which for pointer - * arithmetic is still int, and a four-byte slot truncates the address. - */ - if (!v->type) - return PTR_SIZE; - - /* A direct wide scalar needs both words even when its address never - * escapes: spilling an SSA temporary is still a memory round trip. - */ - if (!is_record_type(v->type) && v->type->size == 8) - return 8; - - /* A global is memory another function reads, and its data slot is only as - * wide as its type: "unsigned int a, b;" places b four bytes after a on - * LP64, so a full-width store to a overwrites b. - */ - if (!v->address_taken && !v->is_global) - return PTR_SIZE; - - /* Classify by the stored width rather than by type identity: an enum is a - * distinct type_t that still stores as a 4-byte int, and a store through an - * enum pointer writes only those 4 bytes. Reading the slot any wider then - * picks up whatever the stack happened to hold above it. - */ - if (v->type->base_type == TYPE_struct) - return PTR_SIZE; - if (v->type->size == 1 || v->type->size == 2 || v->type->size == 4) - return v->type->size; - return PTR_SIZE; -} - -/* High halves stay out of REGS[]: its users assume a register entry names one - * ordinary scalar owner. This parallel reservation table prevents a pair's high - * machine register from being handed to a different value. - */ -var_t *pair_high_owner[REG_CNT]; - -void vreg_map_to_phys(var_t *var, int phys_reg) -{ - if (var) - var->phys_reg = phys_reg; -} - -/* A wide scalar on a 32-bit target owns two independent machine registers. - * Keeping the high mapping on the SSA value, rather than duplicating @var in - * REGS[], lets the existing register file continue to mean one complete - * single-word owner per entry while paired allocation is introduced. - */ -void vreg_map_pair_to_phys(var_t *var, int low_reg, int high_reg) -{ - if (!var) - return; - if (high_reg < 0 || high_reg >= REG_CNT) - fatal("Invalid high register for wide value"); - if (pair_high_owner[high_reg] && pair_high_owner[high_reg] != var) - fatal("Wide value high register is already reserved"); - var->phys_reg = low_reg; - var->phys_reg_hi = high_reg; - pair_high_owner[high_reg] = var; -} - -int vreg_get_phys(var_t *var) -{ - if (var) - return var->phys_reg; - return -1; -} - -int vreg_get_phys_hi(var_t *var) -{ - if (var) - return var->phys_reg_hi; - return -1; -} - -/* ABI argument locations are measured in machine words, not source parameters. - * Keeping that calculation in one place matters once a direct 64-bit scalar - * occupies two words on a 32-bit target: its first word must be even-aligned, - * and every following argument starts after both words. - */ -int abi_arg_words(const var_t *var) -{ - return var_needs_register_pair(var) ? 2 : 1; -} - -/* Where an argument starts, given the next free ABI word. - * - * AAPCS32 starts every long long at an even word, in registers and on the stack - * alike. The RV32 calling convention does so only for a variadic argument and - * on the stack, whose slots are eight-byte aligned; a named one takes the next - * two words, and when only a7 is left its low word goes there and its high word - * to the first stack slot. - */ -int abi_arg_start(int cursor, const var_t *var, bool variadic) -{ - if (abi_arg_words(var) == 2 && - (ELF_MACHINE != 0xf3 || variadic || cursor >= MAX_ARGS_IN_REG)) - cursor = ALIGN_UP(cursor, 2); - return cursor; -} - -int abi_arg_next(int cursor, const var_t *var, bool variadic) -{ - return abi_arg_start(cursor, var, variadic) + abi_arg_words(var); -} - -/* Whether an argument starting at @word is split between the last argument - * register and the stack, which only a named RV32 long long can be. - */ -bool abi_arg_is_split(int word, const var_t *var) -{ - return word < MAX_ARGS_IN_REG && - word + abi_arg_words(var) > MAX_ARGS_IN_REG; -} - -int abi_param_start(const func_t *func, int param_idx) -{ - int cursor = func->returns_aggregate ? 1 : 0; - - for (int i = 0; i < param_idx; i++) - cursor = abi_arg_next(cursor, &func->param_defs[i], false); - return abi_arg_start(cursor, &func->param_defs[param_idx], false); -} - -void vreg_clear_phys(var_t *var) -{ - if (var) { - if (var->phys_reg_hi >= 0 && var->phys_reg_hi < REG_CNT && - pair_high_owner[var->phys_reg_hi] == var) - pair_high_owner[var->phys_reg_hi] = NULL; - var->phys_reg = -1; - var->phys_reg_hi = -1; - } -} - -/* Aligns size to nearest multiple of 4, this meets ARMv7's alignment - * requirement. - * - * This function should be called whenever handling with user-defined type's - * size. - */ -int align_size(int i) -{ - return i <= 4 ? 4 : (i + 3) & ~3; -} - -bool aggregate_has_function_pointer_seen(type_t *type, - type_t **seen, - int seen_count) -{ - if (!type) - return false; - - for (int i = 0; i < seen_count; i++) - if (seen[i] == type) - return false; - - if (seen_count >= MAX_TYPES) - return false; - seen[seen_count++] = type; - - if (type->base_type == TYPE_typedef && type->base_struct) - type = type->base_struct; - - for (int i = 0; i < type->num_fields; i++) { - var_t *field = &type->fields[i]; - if (field->is_func) - return true; - - if (!field->ptr_level && field->type && !field->type->ptr_level && - (field->type->base_type == TYPE_struct || - field->type->base_type == TYPE_union || - field->type->base_type == TYPE_typedef) && - aggregate_has_function_pointer_seen(field->type, seen, seen_count)) - return true; - } - - return false; -} - -bool aggregate_has_function_pointer(type_t *type) -{ - type_t *seen[MAX_TYPES]; - return aggregate_has_function_pointer_seen(type, seen, 0); -} - -bool check_live_out(basic_block_t *bb, var_t *var) -{ - return var_list_holds(&bb->live_out, var); -} - -void track_var_use(var_t *var, int insn_idx) -{ - if (!var) - return; - - var->use_count++; - - if (var->first_use < 0) - var->first_use = insn_idx; - - var->last_use = insn_idx; -} - -void refresh(basic_block_t *bb, const insn_t *insn) -{ - for (int i = 0; i < REG_CNT; i++) { - if (!REGS[i].var) - continue; - if (check_live_out(bb, REGS[i].var)) - continue; - if (REGS[i].var->consumed < insn->idx) { - vreg_clear_phys(REGS[i].var); - REGS[i].var = NULL; - REGS[i].polluted = 0; - } - } -} - -ph2_ir_t *bb_add_ph2_ir(basic_block_t *bb, opcode_t op) -{ - ph2_ir_t *n = arena_alloc(BB_ARENA, sizeof(ph2_ir_t)); - n->op = op; - /* Initialize all fields explicitly */ - n->next = NULL; /* well-formed singly linked list */ - n->is_branch_detached = 0; /* arch-lowering will set for branches */ - n->src0 = 0; - n->src1 = 0; - - /* Only a select names a third source, but the allocation is not zeroed and - * every field is set here by hand. - */ - n->src2 = 0; - n->dest = 0; - n->src0_hi = -1; - n->src1_hi = -1; - n->dest_hi = -1; - n->func_name = NULL; - n->next_bb = NULL; - n->then_bb = NULL; - n->else_bb = NULL; - n->ofs_based_on_stack_top = false; - n->size_bytes = PTR_SIZE; /* default to the full slot; see add_ph2_ir */ - n->is_pointer = false; - n->is_unsigned = false; - n->src0_is_pointer = false; - n->src1_is_pointer = false; - n->src0_is_unsigned = false; - n->src1_is_unsigned = false; - n->is_volatile = false; - - if (!bb->ph2_ir_list.head) - bb->ph2_ir_list.head = n; - else - bb->ph2_ir_list.tail->next = n; - - bb->ph2_ir_list.tail = n; - return n; -} - -/* Calculate the cost of spilling a variable from a register. Higher cost means - * the variable is more valuable to keep in a register. The cost is computed - * based on multiple factors that affect performance. - */ -int calculate_spill_cost(var_t *var, basic_block_t *bb, int current_idx) -{ - int cost = 0; - - /* Variables that are live-out of the basic block must be spilled anyway, so - * give them a high cost to prefer spilling them over others - */ - if (check_live_out(bb, var)) - cost += 1000; - - /* Variables that will be used soon should have higher cost. The closer the - * next use, the higher the penalty for spilling - */ - if (var->consumed > current_idx) { - int distance = var->consumed - current_idx; - if (distance < 10) - cost += 100 - distance * 10; /* Max 100 points for immediate use */ - } - - /* Frequently used variables should stay in registers. Each use adds 5 - * points to the cost - */ - if (var->use_count > 0) - cost += var->use_count * 5; - - /* Constants can be easily reloaded, so prefer spilling them by reducing - * their cost - */ - if (var->is_const) - cost -= 50; - - /* Variables with long live ranges may benefit from spilling to free up - * registers for other variables - */ - if (var->first_use >= 0 && var->last_use >= 0) { - int range_length = var->last_use - var->first_use; - if (range_length > 100) - cost += 20; /* Small penalty for very long live ranges */ - } - - return cost; -} - -/* Variables kept in a register for the whole of the current function, named by - * the base each SSA version was renamed from. - * - * The base is the right unit. Every assignment makes a new SSA version, so - * pinning versions individually holds nothing: each one dies within a few - * instructions and the value still travels between them through a stack slot. - * Pinning the base keeps the variable itself in a register, which is what a - * loop carrying it across an iteration actually needs. - * - * Only functions that call nothing qualify, because every register in the file - * is caller-saved. - */ -var_t *pinned_base[REG_CNT]; - -/* Registers the instruction being lowered has already placed an operand in. - * - * Preparing one operand can spill another's register to make room, leaving the - * number recorded for it naming a register that no longer holds the value. An - * instruction reading more operands than the two "avoid" arguments can express - * -- a select reads three -- locks each as it is placed, and the spill searches - * leave those alone. - */ -int reg_locked; - -/* ABI argument registers already staged for the call being lowered. A staged - * register can lose its REGS[] owner while the call still needs its value: a - * variable passed twice as a pair, as in "f(g, g, ...)", is moved to its second - * argument pair, which released the first, and the next argument was loaded - * straight over it. - */ -int abi_args_staged; - -/* Whether @reg holds an operand of the instruction being lowered, or an - * argument of the call being staged. - */ -bool reg_is_locked(int reg) -{ - if (reg < 0 || reg >= REG_CNT) - return false; - return ((reg_locked | abi_args_staged) >> reg) & 1; -} - -/* The register @var's base is pinned to, or -1. */ -int pinned_reg_of(const var_t *var) -{ - if (!var || !var->base) - return -1; - - /* A value an if-converted arm computes shares its variable's base with the - * select's result, but the pinned register still holds what the arms read, - * so it takes an ordinary register instead. - */ - if (var->in_select_arm) - return -1; - for (int i = 0; i < REG_CNT; i++) { - if (pinned_base[i] && pinned_base[i] == var->base) - return i; - } - return -1; -} - -int find_best_spill(basic_block_t *bb, - int current_idx, - int avoid_reg1, - int avoid_reg2) -{ - int best_reg = -1; - int min_cost = 99999; - - for (int i = 0; i < REG_CNT; i++) { - if (i == avoid_reg1 || i == avoid_reg2) - continue; - - if (!REGS[i].var || pinned_base[i] || reg_is_locked(i)) - continue; - - int cost = calculate_spill_cost(REGS[i].var, bb, current_idx); - - if (cost < min_cost) { - min_cost = cost; - best_reg = i; - } - } - - return best_reg; -} - -/* Priority of spilling: - * - live_out variable - * - farthest local variable Slots reg_alloc() hands out inside the current - * function, in allocation order and therefore ascending by offset: both places - * that call slot_var_track() assign var->offset = stack_size and then grow the - * frame. Only these are candidates for the two cleanups below; every other - * address in the frame belongs to something a pointer may legally reach. - * - * The parallel arrays are indexed by position in slot_vars and are filled once - * per function by slot_scan(). - */ -var_t *slot_vars[MAX_LOCALS]; -int slot_var_count; -char slot_private[MAX_LOCALS]; -char slot_stores[MAX_LOCALS]; /* saturates at 2: only "exactly one" matters */ -char slot_loads[MAX_LOCALS]; -basic_block_t *slot_home[MAX_LOCALS]; - -/* The first store to and the first load from each slot, and the list node that - * precedes the store. collapse_slot_roundtrip() needs all three, and - * slot_scan() already walks past them, so recording them here saves it a search - * from the head of the block for every slot it considers. - */ -ph2_ir_t *slot_store_ir[MAX_LOCALS]; -ph2_ir_t *slot_store_prev[MAX_LOCALS]; -ph2_ir_t *slot_load_ir[MAX_LOCALS]; - -void slot_var_track(var_t *var) -{ - if (slot_var_count < MAX_LOCALS) - slot_vars[slot_var_count++] = var; -} - -/* Whether the slot's address cannot have escaped, so that the stores and loads - * naming it are the only accesses to it. address_taken covers &var; array_size - * and has_backing_storage cover aggregates whose interior is reached by pointer - * arithmetic. A store through a pointer is invisible to these scans, so - * anything else in the frame is left alone. - */ -bool slot_is_private(const var_t *var) -{ - if (var->address_taken || var->array_size || var->has_backing_storage || - var_is_volatile_object(var)) - return false; - if (var->is_global || var->ofs_based_on_stack_top) - return false; - return true; -} - -/* Position of the slot at @offset in slot_vars, or -1 when the offset does not - * name one. The array is sorted, so this is a binary search. - */ -int slot_lookup(int offset) -{ - int lo = 0, hi = slot_var_count - 1; - - while (lo <= hi) { - int mid = (lo + hi) >> 1; - int at = slot_vars[mid]->offset; - - if (at == offset) - return mid; - if (at < offset) - lo = mid + 1; - else - hi = mid - 1; - } - return -1; -} - -/* Tally the accesses to every tracked slot in one walk of the function. - * - * Both cleanups below need to know how often a slot is stored and loaded. - * Asking that question per slot means re-walking the whole function once for - * each of them, which is quadratic and, on shecc's own larger functions, was - * the single most expensive thing the compiler did. - */ -void slot_scan(func_t *func) -{ - /* alloc_var_slot() appends in increasing offset order, but phi_slot_merge() - * then rewrites the offsets of variables already in the table to put a - * chain of phis on one slot, which can leave it out of order. slot_lookup() - * binary-searches, so an unsorted table makes it miss a slot that is - * present: both scans below and dead_store_elim() miss the same one, so - * nothing is misidentified, but the two cleanups skip work they could do. - * - * Insertion sort, because the table is nearly sorted already and every - * entry a merge moved sits close to where it belongs. It costs about 0.085% - * of a self-compile and recovers optimizations worth rather less than that; - * it is here to keep the invariant alloc_var_slot() documents true, not to - * pay for itself. - */ - for (int i = 1; i < slot_var_count; i++) { - var_t *var = slot_vars[i]; - int j = i - 1; - - while (j >= 0 && slot_vars[j]->offset > var->offset) { - slot_vars[j + 1] = slot_vars[j]; - j--; - } - slot_vars[j + 1] = var; - } - - for (int i = 0; i < slot_var_count; i++) { - slot_private[i] = slot_is_private(slot_vars[i]); - slot_stores[i] = 0; - slot_loads[i] = 0; - slot_home[i] = NULL; - slot_store_ir[i] = NULL; - slot_store_prev[i] = NULL; - slot_load_ir[i] = NULL; - } - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - ph2_ir_t *prev = NULL; - for (ph2_ir_t *ir = bb->ph2_ir_list.head; ir; ir = ir->next) { - if (ir->ofs_based_on_stack_top) { - prev = ir; - continue; - } - if (ir->op == OP_store) { - int i = slot_lookup(ir->src1); - if (i < 0) { - prev = ir; - continue; - } - if (slot_stores[i] < 2) { - if (!slot_stores[i]) { - slot_store_ir[i] = ir; - slot_store_prev[i] = prev; - } - slot_stores[i]++; - } - slot_home[i] = bb; - } else if (ir->op == OP_load) { - int i = slot_lookup(ir->src0); - if (i < 0) { - prev = ir; - continue; - } - if (slot_loads[i] < 2) { - if (!slot_loads[i]) - slot_load_ir[i] = ir; - slot_loads[i]++; - } - if (slot_home[i] != bb) - slot_home[i] = NULL; - } - prev = ir; - } - } -} - -/* Unlink @ir, whose predecessor in @bb's list is @prev (NULL at the head). */ -void ph2_list_remove(basic_block_t *bb, ph2_ir_t *prev, ph2_ir_t *ir) -{ - if (prev) - prev->next = ir->next; - else - bb->ph2_ir_list.head = ir->next; - if (bb->ph2_ir_list.tail == ir) - bb->ph2_ir_list.tail = prev; - - /* Clearing the link marks the node as gone. One left pointing at its old - * successor still looks like that successor's predecessor, and - * collapse_slot_roundtrip() decides whether the predecessor it recorded is - * still usable by asking exactly that. - */ - ir->next = NULL; -} - -/* Whether @ir leaves @reg holding something other than what it held before. */ -bool ph2_writes_reg(const ph2_ir_t *ir, int reg) -{ - switch (ir->op) { - case OP_store: - case OP_global_store: - case OP_write: - case OP_branch: - case OP_jump: - case OP_return: - case OP_push: - return false; - case OP_call: - case OP_indirect: - return true; /* the call clobbers the caller-saved registers */ - default: - /* A pair result writes its high register too. */ - return ir->dest == reg || (reg >= 0 && ir->dest_hi == reg); - } -} - -/* Whether @ir clobbers either register of the value stored from @store. */ -bool ph2_writes_stored_value(const ph2_ir_t *ir, const ph2_ir_t *store) -{ - return ph2_writes_reg(ir, store->src0) || - (store->src0_hi >= 0 && ph2_writes_reg(ir, store->src0_hi)); -} - -/* Collapse a value that goes out to a stack slot and comes straight back. - * - * Unwinding a phi writes the value to the phi's own slot, and the next - * instruction reads it back to copy it into the variable's slot -- a round trip - * through memory that also hides the "s = s + x" pattern from - * try_fold_alu_to_slot(), which would otherwise emit a single add to the slot. - * When the slot is written once and read once in the same block, and the source - * register survives in between, the load is just a register move and the store - * is dead. - */ -void collapse_slot_roundtrip(func_t *func) -{ - UNUSED(func); - for (int i = 0; i < slot_var_count; i++) { - if (!slot_private[i] || slot_stores[i] != 1 || slot_loads[i] != 1) - continue; - - basic_block_t *home = slot_home[i]; - if (!home) - continue; - - int offset = slot_vars[i]->offset; - ph2_ir_t *store = slot_store_ir[i], *load = slot_load_ir[i]; - ph2_ir_t *prev_store = slot_store_prev[i], *prev_load = NULL; - bool ok = false; - - /* slot_scan() already walked past this slot's only store and its only - * load, so the search that used to start at the head of the block is - * needed only when an earlier collapse in this same block unlinked the - * node the store was recorded behind. A removed node still points at - * its old successor, which is why the check is that the recorded - * predecessor is still in the list rather than that it still points - * here: ph2_list_remove() clears the link of whatever it takes out. - */ - bool linked = store && load; - if (linked) { - if (prev_store) { - if (prev_store->next != store) - linked = false; - } else if (home->ph2_ir_list.head != store) - linked = false; - } - - if (linked) { - /* The store has to come first, and its register has to still hold - * the value when the load would have run. Walking from the store to - * the load settles both: a load placed ahead of the store is never - * reached, and a clobber on the way stops the walk short. The walk - * passes the load's predecessor, so that comes from here rather - * than from a search. - */ - ph2_ir_t *prev = store; - - for (ph2_ir_t *ir = store->next; ir; ir = ir->next) { - if (ir == load) { - prev_load = prev; - ok = true; - break; - } - if (ph2_writes_stored_value(ir, store)) - break; - prev = ir; - } - } else { - ph2_ir_t *prev = NULL; - - store = NULL; - load = NULL; - prev_store = NULL; - for (ph2_ir_t *ir = home->ph2_ir_list.head; ir; ir = ir->next) { - if (!ir->ofs_based_on_stack_top) { - if (!store && ir->op == OP_store && ir->src1 == offset) { - store = ir; - prev_store = prev; - prev = ir; - continue; - } - if (ir->op == OP_load && ir->src0 == offset) { - load = ir; - prev_load = prev; - ok = store != NULL; - break; - } - } - if (store && ph2_writes_stored_value(ir, store)) - break; - prev = ir; - } - } - - /* A pair is moved as a pair: forwarding only the low register turned - * the load into a move that left the high register unwritten. - */ - if (!ok || (store->src0_hi >= 0) != (load->dest_hi >= 0)) - continue; - - load->op = OP_assign; - load->src0 = store->src0; - load->src0_hi = store->src0_hi; - - /* Removing the store first would leave prev_load stale when the store - * is the load's predecessor, so drop the later one first. - */ - if (load->dest == load->src0 && load->dest_hi == load->src0_hi) - ph2_list_remove(home, prev_load, load); - ph2_list_remove(home, prev_store, store); - } -} - -/* Drop stores to a slot that nothing in the function ever loads. - * - * Unwinding a phi copies the value into the phi's slot, and the source SSA temp - * is written to a slot of its own that no one reads -- one wasted store per phi - * operand, on every iteration of a loop. - */ -void dead_store_elim(func_t *func) -{ - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - ph2_ir_t *prev = NULL, *next; - for (ph2_ir_t *ir = bb->ph2_ir_list.head; ir; ir = next) { - next = ir->next; - if (ir->op == OP_store && !ir->ofs_based_on_stack_top) { - int i = slot_lookup(ir->src1); - if (i >= 0 && slot_private[i] && !slot_loads[i]) { - ph2_list_remove(bb, prev, ir); - continue; - } - } - prev = ir; - } - } -} - -/* Write the register back to the variable's stack slot, allocating the slot on - * first use. The register keeps holding the value; only memory is made - * coherent, so callers decide separately whether to drop the association. - * Reserve @var's stack slot in @func's frame. - * - * A variable whose address escapes is reached through a pointer that carries no - * information about what it points to, so give it the alignment any object type - * could need rather than only its own. Slots stay in increasing offset order, - * which slot_lookup()'s binary search relies on. - */ -void alloc_var_slot(func_t *func, var_t *var) -{ - int slot_size = var_slot_size(var); - - if (var->address_taken) - func->stack_size = ALIGN_UP(func->stack_size, 16); - - /* A future 32-bit wide scalar occupies a low/high word pair. It must not - * start at an arbitrary four-byte offset: both AAPCS32 and the RV32 ABI - * require the pair's stack home to be eight-byte aligned. - */ - if (slot_size > PTR_SIZE) - func->stack_size = ALIGN_UP(func->stack_size, slot_size); - var->offset = func->stack_size; - var->space_is_allocated = true; - func->stack_size += slot_size; - slot_var_track(var); -} - -void store_var(basic_block_t *bb, var_t *var, int idx) -{ - if (!var->space_is_allocated) { - alloc_var_slot(bb->belong_to, var); - } - ph2_ir_t *ir = var->is_global ? bb_add_ph2_ir(bb, OP_global_store) - : bb_add_ph2_ir(bb, OP_store); - ir->src0 = idx; - ir->src0_hi = vreg_get_phys_hi(var); - ir->src1 = var->offset; - ir->ofs_based_on_stack_top = var->ofs_based_on_stack_top; - ir->is_pointer = is_pointer_like(var); - ir->is_unsigned = is_unsigned_scalar(var); - ir->size_bytes = var_slot_size(var); - ir->is_volatile = var_is_volatile_object(var); - REGS[idx].polluted = 0; -} - -void spill_var(basic_block_t *bb, var_t *var, int idx) -{ - if (REGS[idx].polluted) - store_var(bb, var, idx); - REGS[idx].var = NULL; - REGS[idx].polluted = 0; - vreg_clear_phys(var); -} - -/* Whether register @i can be handed to a value that wants one. - * - * Empty is not enough: a pinned register looks empty whenever the version it - * currently holds has been consumed, but nothing ever reloads it and the next - * read of its variable expects to find the value still there. - */ -bool reg_is_free(int i) -{ - return !REGS[i].var && !pinned_base[i] && !pair_high_owner[i] && - !((abi_args_staged >> i) & 1); -} - -/* Return the index of register for given variable. Otherwise, return -1. */ -int find_in_regs(const var_t *var) -{ - for (int i = 0; i < REG_CNT; i++) { - if (REGS[i].var == var) - return i; - } - return -1; -} - -/* Whether @var can live in a register for a whole function: nothing else may be - * able to reach it, and it must fit in one register. - */ -bool var_is_pinnable(var_t *var) -{ - if (!var || var->is_const || !var->base) - return false; - if (var_is_volatile_object(var)) - return false; - - /* slot_is_private() rules out everything reachable other than by name: - * globals, address-taken variables, arrays and backing storage. - */ - if (!slot_is_private(var)) - return false; - return var_slot_size(var) <= PTR_SIZE; -} - -/* Choose which variables this function keeps in registers. - * - * Candidates are ranked by how often they are named, which stands in well - * enough for how hot they are: a variable a loop carries is named on every - * iteration. At most half the file is given away so expression evaluation still - * has registers to work with. - */ -int pin_scan_gen; - -void pin_registers(func_t *func) -{ - int limit = REG_CNT / 2; - bool calls = false; - bool pairs = false; - - for (int i = 0; i < REG_CNT; i++) - pinned_base[i] = NULL; - func->pinned_regs = 0; - - mark_loop_depth(func); - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->opcode == OP_call || insn->opcode == OP_indirect || - insn->opcode == OP_push) - calls = true; - if (!pairs && (var_needs_register_pair(insn->rd) || - var_needs_register_pair(insn->rs1) || - var_needs_register_pair(insn->rs2))) - pairs = true; - /* Taking an address makes the frame reachable by other means. */ - if (insn->opcode == OP_address_of) - return; - } - } - - /* A binary operation on register pairs holds two operand pairs and a - * destination pair at once. Pinning half the file left a 32-bit target with - * too few registers to lower one, so keep six of them unpinned. - */ - if (pairs && limit > REG_CNT - 6) - limit = REG_CNT - 6; - - /* Across a call only the registers the callee preserves will still hold - * their value. Those sit at the top of the file, which is the end the loop - * below hands out from, so capping the count is all that is needed. - */ - if (calls) { - if (!CALLEE_SAVED_REGS) - return; - if (limit > CALLEE_SAVED_REGS) - limit = CALLEE_SAVED_REGS; - } - - /* Tally what every variable's namings are worth. The count lives on the - * variable rather than in an array here: the file has a handful of - * registers and a function names hundreds of variables, and the one worth a - * register is not reliably among the first few met -- a pointer - * strength_reduce() introduced is named last of all. - */ - pin_scan_gen++; - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - var_t *ops[4]; - - ops[0] = insn->rd; - ops[1] = insn->rs1; - ops[2] = insn->rs2; - ops[3] = insn->rs3; - - for (int k = 0; k < 4; k++) { - var_t *var = ops[k]; - - if (!var_is_pinnable(var)) - continue; - - /* A parameter passed on the stack lives at an offset into the - * caller's frame that the callee cannot pin; one passed in a - * register is moved into its pinned register on entry, so it is - * a candidate like any other -- and a good one, since a pointer - * a loop walks is usually a parameter. - */ - bool on_stack = false; - bool in_reg = false; - - for (int q = 0; q < func->num_params; q++) { - if (var->base != &func->param_defs[q]) - continue; - int word = abi_param_start(func, q); - if (word + abi_arg_words(&func->param_defs[q]) <= - MAX_ARGS_IN_REG) - in_reg = true; - else - on_stack = true; - } - if (on_stack) - continue; - - var_t *base = var->base; - - if (base->pin_gen != pin_scan_gen) { - base->pin_gen = pin_scan_gen; - base->pin_weight = 0; - - /* A parameter is written in the entry block before the body - * names it, so its range already spans blocks. - */ - base->pin_cross = in_reg; - base->pin_hot = false; - base->pin_blk = -1; - } - base->pin_weight += bb->loop_weight; - if (base->pin_blk >= 0 && base->pin_blk != bb->rpo) - base->pin_cross = true; - base->pin_blk = bb->rpo; - if (bb->loop_depth) - base->pin_hot = true; - } - } - } - - /* Take the most-named bases, highest register first: the low registers are - * where arguments and return values land. The winner is found by walking - * the tally again rather than sorting it, which costs one pass per register - * handed out -- a handful, against a function's instruction count. - */ - for (int taken = 0; taken < limit; taken++) { - var_t *best = NULL; - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - var_t *ops[4]; - - ops[0] = insn->rd; - ops[1] = insn->rs1; - ops[2] = insn->rs2; - ops[3] = insn->rs3; - - for (int k = 0; k < 4; k++) { - if (!ops[k] || !ops[k]->base) - continue; - - var_t *base = ops[k]->base; - - if (base->pin_gen != pin_scan_gen) - continue; - - /* Registers are handed out from the top of the file, which - * is where the preserved ones are, so a pinned function - * saves and restores one in its prologue. A variable named - * once in straight-line code does not earn that back -- - * least of all in a small leaf called from a loop, which - * pays it on every call -- so a candidate has to be read - * inside a loop and to outlive the block that names it. - */ - if (!base->pin_cross || !base->pin_hot) - continue; - - bool already = false; - - for (int r = 0; r < REG_CNT; r++) { - if (pinned_base[r] == base) - already = true; - } - if (already) - continue; - if (!best || base->pin_weight > best->pin_weight) - best = base; - } - } - } - if (!best) - return; - - /* Incoming arguments arrive in the low registers and are placed there - * before the body runs, which would overwrite anything pinned to one of - * them; the variable would then read a parameter's value instead. - */ - int reg = REG_CNT - 1 - taken; - int arg_words_in_reg = func->returns_aggregate ? 1 : 0; - for (int i = 0; i < func->num_params; i++) { - int word = abi_param_start(func, i); - - if (word + abi_arg_words(&func->param_defs[i]) > MAX_ARGS_IN_REG) - break; - arg_words_in_reg = abi_arg_next(word, &func->param_defs[i], false); - } - - if (reg < arg_words_in_reg) - return; - - pinned_base[reg] = best; - func->pinned_regs = func->pinned_regs | (1 << reg); - } -} - -/* Defined beside OP_push handling below; allocation must not reclaim ABI - * argument registers once argument staging has begun. - */ -extern bool is_pushing_args; - -void load_var(basic_block_t *bb, var_t *var, int idx) -{ - ph2_ir_t *ir; - - /* Load constants directly, others from memory */ - if (var->is_const) { - ir = bb_add_ph2_ir(bb, OP_load_constant); - ir->src0 = var->init_val; - ir->src1 = var->init_val_hi; - } else if (var->is_global && var->array_size) { - /* A global array's address is fixed for the life of the program, and - * the initialiser recorded where its storage sits. Computing it beats - * loading back the pointer that was stored there: the address is an - * addition rather than a memory read, so it does not have to wait on - * the data cache on every trip round a loop. - */ - ir = bb_add_ph2_ir(bb, OP_global_address_of); - ir->src0 = var->init_val; - } else { - ir = var->is_global ? bb_add_ph2_ir(bb, OP_global_load) - : bb_add_ph2_ir(bb, OP_load); - ir->src0 = var->offset; - ir->ofs_based_on_stack_top = var->ofs_based_on_stack_top; - ir->is_volatile = var_is_volatile_object(var); - } - - ir->dest = idx; - ir->dest_hi = vreg_get_phys_hi(var); - ir->is_pointer = is_pointer_like(var); - ir->is_unsigned = is_unsigned_scalar(var); - - /* Incoming stack arguments use ABI-sized slots but carry a scalar in the - * low declared-width bytes. Reloading all eight bytes can retain a caller's - * sign extension; use the declaration width to preserve unsigned argument - * semantics. - */ - ir->size_bytes = var->ofs_based_on_stack_top && !var->ptr_level && - !var->is_func && var->type - ? var->type->size - : var_slot_size(var); - REGS[idx].var = var; - REGS[idx].polluted = 0; - vreg_map_to_phys(var, idx); -} - -/* The two lowest free registers, in *@low and *@high. */ -bool find_free_pair(int *low, int *high) -{ - *low = -1; - *high = -1; - for (int r = 0; r < REG_CNT; r++) { - if (!reg_is_free(r)) - continue; - if (*low < 0) { - *low = r; - } else { - *high = r; - return true; - } - } - return false; -} - -/* Two free registers for a register pair, in *@low and *@high, spilling to make - * room when the file is full. The ordinary chooser frees one register, which - * cannot help a pair, so spill whole pairs through their slots first, which - * keeps both words, and then one-word values, whether live or dead. Neither - * @operand_0 nor @operand_1 is touched, nor a pinned register, and a one-word - * spill also leaves reserved high halves and locked inputs alone. Once OP_push - * has installed ABI arguments, those registers are live until the call, so - * nothing is spilled then. - */ -bool claim_free_pair(basic_block_t *bb, - int operand_0, - int operand_1, - int *low, - int *high) -{ - if (find_free_pair(low, high) || is_pushing_args) - return *high >= 0; - - for (int scalars = 0; scalars < 2; scalars++) { - for (int r = 0; r < REG_CNT; r++) { - var_t *owner = REGS[r].var; - - if (!owner || r == operand_0 || r == operand_1 || pinned_base[r]) - continue; - if (scalars) { - if (var_needs_register_pair(owner) || pair_high_owner[r] || - reg_is_locked(r)) - continue; - } else if (!var_needs_register_pair(owner)) { - continue; - } - spill_var(bb, owner, r); - if (find_free_pair(low, high)) - return true; - } - } - return false; -} - -int prepare_operand(basic_block_t *bb, var_t *var, int operand_0) -{ - /* A pinned variable is already where it always is -- unless this version of - * it is a constant, which has no home to be in until it is written there. - * Every other version reaches the register by being defined into it or by a - * phi move, so nothing else needs materialising. - */ - int pinned = pinned_reg_of(var); - if (pinned >= 0) { - if (var->is_const && REGS[pinned].var != var) - load_var(bb, var, pinned); - REGS[pinned].var = var; - return pinned; - } - - /* Check VReg mapping first for O(1) lookup. A register holding a volatile - * variable only as its storage last held it is left to the reload below. - */ - int phys_reg = vreg_get_phys(var); - if (phys_reg >= 0 && phys_reg < REG_CNT && REGS[phys_reg].var == var) { - if (!var_is_volatile_object(var) || REGS[phys_reg].polluted) - return phys_reg; - - /* Reload a volatile variable into the register, or the pair, that - * already names it. Loading it into others left two registers naming - * the variable, and store_addressed_def(), which finds the first, took - * the stale one for the one a later assignment wrote and skipped the - * store. - */ - if (phys_reg != operand_0) { - load_var(bb, var, phys_reg); - return phys_reg; - } - } - - if (var_needs_register_pair(var)) { - int low, high; - - if (!claim_free_pair(bb, operand_0, -1, &low, &high)) - fatal("Wide operand needs two free registers"); - vreg_map_pair_to_phys(var, low, high); - load_var(bb, var, low); - return low; - } - - /* Force reload for address-taken variables (may be modified via pointer). - * - * Not when the register holds a value the slot never received, though. No - * pointer can have reached the slot since: taking the address stores the - * register first, and a write through a pointer or a call spills or drops - * every register. A parameter still in the register it arrived in is that - * case, and its slot may not exist yet, so reloading it read whatever sat - * at offset zero of the frame instead. - * - * A volatile global is reloaded for the same reason and one more: every - * read of it is an access the program has to perform (C99 6.7.3p6), not - * just a way to learn its value. A volatile local is address-taken already. - */ - int i = find_in_regs(var); - if (i > -1 && (!(var->address_taken || var_is_volatile_object(var)) || - REGS[i].polluted)) { - vreg_map_to_phys(var, i); - return i; - } - - /* As above, when the register lost the mapping but still names it. */ - if (i > -1 && i != operand_0 && var_is_volatile_object(var)) { - load_var(bb, var, i); - return i; - } - - for (i = 0; i < REG_CNT; i++) { - if (reg_is_free(i)) { - load_var(bb, var, i); - vreg_map_to_phys(var, i); - return i; - } - } - - int spilled = find_best_spill( - bb, bb->insn_list.tail ? bb->insn_list.tail->idx : 0, operand_0, -1); - - if (spilled < 0) { - for (i = 0; i < REG_CNT; i++) { - /* A pinned register is not available: nothing reloads it, so - * handing it to another variable loses the pinned value. - */ - if (i != operand_0 && REGS[i].var && !pinned_base[i] && - !reg_is_locked(i)) { - spilled = i; - break; - } - } - } - - /* spill_var() forgets the mapping once it has stored the value. Clearing it - * first dropped a pair's high register, so only the low word was stored. - */ - spill_var(bb, REGS[spilled].var, spilled); - load_var(bb, var, spilled); - vreg_map_to_phys(var, spilled); - - return spilled; -} - -/* Set while lowering the run of OP_push that stages a call's arguments. The - * pushed values have to stay put until the call consumes them, and that - * liveness is recorded on var->consumed rather than in the block's remaining - * instructions, so no register may be reused for anything here. - */ -bool is_pushing_args; - -/* Whether @var is read again before the end of @bb. - * - * var->consumed is the last use anywhere in the function, which says nothing - * about this block: a value whose only remaining reader sits on a path this - * block does not reach is dead here, and every value that outlives the block is - * in its live-out set. Asking the block directly is what makes the two arms of - * an if-else able to reuse the same register. - */ -bool var_read_later_in_bb(basic_block_t *bb, insn_t *from, const var_t *var) -{ - UNUSED(bb); - for (insn_t *insn = from; insn; insn = insn->next) { - if (insn->rs1 == var || insn->rs2 == var || insn->rs3 == var) - return true; - } - return false; -} - -/* Forget every register a call does not preserve. - * - * A pinned register the callee preserves still holds its variable, and - * forgetting that would leave it looking free for the allocator to hand to - * something else, losing the value the call went to the trouble of keeping. - */ -void clobber_caller_saved(void) -{ - for (int i = 0; i < REG_CNT; i++) { - if (!pinned_base[i] && REGS[i].var) { - vreg_clear_phys(REGS[i].var); - REGS[i].var = NULL; - } - if (!REGS[i].var) - REGS[i].polluted = 0; - } -} - -/* x86 ALU instructions are two-operand: "rd = rs1 OP rs2" is emitted as "MOV - * rd, rs1" followed by "OP rd, rs2", and the MOV disappears entirely when rd - * already names rs1's register. So when a source operand is dead after this - * instruction, its register is the best possible home for the destination. - * - * Only a plain local qualifies. A global or an address-taken variable is - * reachable through memory, so its register cannot simply be repurposed even - * once the SSA name is dead. - */ -int coalesce_candidate(basic_block_t *bb, insn_t *insn, int reg) -{ - if (reg < 0 || reg >= REG_CNT) - return -1; - - if (!insn) - return -1; - - /* Restricted to the pure ALU forms, whose lowering is exactly "MOV rd, rs1; - * OP rd, rs2". Everything else -- loads, calls, address arithmetic the - * backend folds into addressing modes -- has its own register expectations - * and is left alone. - */ - switch (insn->opcode) { - case OP_add: - case OP_sub: - case OP_mul: - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: - case OP_lshift: - case OP_rshift: - case OP_negate: - case OP_bit_not: - break; - default: - return -1; - } - - if (is_pushing_args) - return -1; - - /* A pinned register is not a scratch one however dead the version it - * currently holds looks: the variable living there is read again further on - * and nothing ever reloads it. Handing it to a temporary destroyed the - * value a select was about to choose between. - */ - if (pinned_base[reg]) - return -1; - - var_t *v = REGS[reg].var; - if (!v || v->is_global || v->address_taken) - return -1; - if (check_live_out(bb, v)) - return -1; - if (var_read_later_in_bb(bb, insn->next, v)) - return -1; - return reg; -} - -int prepare_dest(basic_block_t *bb, - insn_t *insn, - var_t *var, - int operand_0, - int operand_1) -{ - if (var_needs_register_pair(var)) { - int mapped_low = vreg_get_phys(var); - int mapped_high = vreg_get_phys_hi(var); - if (mapped_low >= 0 && mapped_low < REG_CNT && mapped_high >= 0 && - mapped_high < REG_CNT && REGS[mapped_low].var == var && - pair_high_owner[mapped_high] == var) { - REGS[mapped_low].polluted = 1; - return mapped_low; - } - int low, high; - - if (!claim_free_pair(bb, operand_0, operand_1, &low, &high)) - fatal("Wide destination needs two free registers"); - REGS[low].var = var; - REGS[low].polluted = 1; - vreg_map_pair_to_phys(var, low, high); - return low; - } - - int pinned = pinned_reg_of(var); - if (pinned >= 0) { - REGS[pinned].var = var; - REGS[pinned].polluted = 1; - return pinned; - } - - int phys_reg = vreg_get_phys(var); - if (phys_reg >= 0 && phys_reg < REG_CNT && REGS[phys_reg].var == var) { - REGS[phys_reg].polluted = 1; - return phys_reg; - } - - int i = find_in_regs(var); - if (i > -1) { - REGS[i].polluted = 1; - vreg_map_to_phys(var, i); - return i; - } - - /* Reuse a dying source operand's register so the two-operand lowering needs - * no MOV. Taking a free register instead would cost one on every such - * instruction. - */ - i = coalesce_candidate(bb, insn, operand_0); - if (i < 0) - i = coalesce_candidate(bb, insn, operand_1); - if (i > -1) { - vreg_clear_phys(REGS[i].var); - REGS[i].var = var; - REGS[i].polluted = 1; - vreg_map_to_phys(var, i); - return i; - } - - for (i = 0; i < REG_CNT; i++) { - if (reg_is_free(i)) { - REGS[i].var = var; - REGS[i].polluted = 1; - vreg_map_to_phys(var, i); - return i; - } - } - - int spilled = - find_best_spill(bb, bb->insn_list.tail ? bb->insn_list.tail->idx : 0, - operand_0, operand_1); - - if (spilled < 0) { - for (i = 0; i < REG_CNT; i++) { - if (i != operand_0 && i != operand_1 && REGS[i].var && - !pinned_base[i] && !reg_is_locked(i)) { - spilled = i; - break; - } - } - } - - /* Callers normally have at least one register which is not an operand, but - * an instruction with more register inputs than the allocator's two avoid - * arguments can leave every register protected. Let that caller make an - * instruction-specific choice instead of indexing REGS[-1]. - */ - if (spilled < 0) - return -1; - - /* As in prepare_operand(), spill before forgetting the mapping, or a pair's - * high word is not stored. - */ - spill_var(bb, REGS[spilled].var, spilled); - REGS[spilled].var = var; - REGS[spilled].polluted = 1; - vreg_map_to_phys(var, spilled); - - return spilled; -} - -void spill_alive(basic_block_t *bb, const insn_t *insn) -{ - /* Spill every variable on pointer writes (conservative aliasing handling). - * A global is no exception: the pointer can hold its address, and a copy - * left in a register is what the next read of it would use. - */ - if (insn && insn->opcode == OP_write) { - for (int i = 0; i < REG_CNT; i++) { - /* A pinned variable has no address, so no write through a pointer - * can reach it. - */ - if (REGS[i].var && !pinned_base[i]) - spill_var(bb, REGS[i].var, i); - } - return; - } - - /* Standard spilling for non-pointer operations */ - for (int i = 0; i < REG_CNT; i++) { - if (!REGS[i].var || pinned_base[i]) - continue; - if (check_live_out(bb, REGS[i].var)) { - spill_var(bb, REGS[i].var, i); - continue; - } - if (REGS[i].var->consumed > insn->idx) { - spill_var(bb, REGS[i].var, i); - continue; - } - } -} - -void spill_live_out(basic_block_t *bb) -{ - for (int i = 0; i < REG_CNT; i++) { - if (!REGS[i].var || pinned_base[i]) - continue; - if (!check_live_out(bb, REGS[i].var)) { - vreg_clear_phys(REGS[i].var); - REGS[i].var = NULL; - REGS[i].polluted = 0; - continue; - } - if (!var_check_killed(REGS[i].var, bb)) { - vreg_clear_phys(REGS[i].var); - REGS[i].var = NULL; - REGS[i].polluted = 0; - continue; - } - spill_var(bb, REGS[i].var, i); - } -} - -/* End a block the way spill_live_out() does -- every live-out variable written - * here is flushed to its stack slot, so memory is coherent for every path out - * of 'bb' -- but keep the register associations instead of dropping them. What - * survives the boundary is decided per successor by bb_export_regs(). - */ -void spill_live_out_keep(basic_block_t *bb) -{ - for (int i = 0; i < REG_CNT; i++) { - var_t *var = REGS[i].var; - if (!var || pinned_base[i]) - continue; - if (!check_live_out(bb, var)) { - vreg_clear_phys(var); - REGS[i].var = NULL; - REGS[i].polluted = 0; - continue; - } - if (REGS[i].polluted) - store_var(bb, var, i); - } -} - -/* Hand the register file to the successor when it is the only way out of 'bb' - * and the only way in to the successor: control then reaches it with exactly - * these registers, so it can read them in place rather than reloading from the - * slots just written. - * - * Requiring the successor to be next in reverse post-order is what makes - * "emitted immediately after" true: cfg_flatten() walks the same rpo_next chain - * the allocator does, so the two orders are the same traversal. - * - * Only a successor emitted immediately after this block qualifies. A branch - * target is emitted wherever the backend's linear walk puts it, and a block the - * walk misses is re-emitted later -- that second copy is reached with unrelated - * registers, so a file handed to it would not hold on every path that runs its - * code. Requiring the successor to be the next block in reverse post-order is - * what rules that out, since that is the order the walk emits. - * - * A conditional branch falls through as well: the backend jumps to one - * successor and lets control run on into the other, which is emitted - * contiguously exactly as a plain successor is. That edge therefore qualifies - * on the same terms, and it is the one that matters -- it is where an if/else - * chain would otherwise reload on every arm what the test just had in a - * register. - */ -void bb_export_regs(basic_block_t *bb) -{ - basic_block_t *succ = bb->next; - - if (!succ) { - /* Take whichever arm the walk placed next; the checks below confirm it - * really is contiguous and has no other way in. - */ - if (bb->then_ == bb->rpo_next) - succ = bb->then_; - else if (bb->else_ == bb->rpo_next) - succ = bb->else_; - else - return; - } - - /* The successor must also be the block reg_alloc() visits next, so that the - * file it inherits is the one just built here. - */ - if (succ != bb->rpo_next || succ->entry_regs) - return; - if (bb_pred_count(succ) != 1) - return; - - succ->entry_regs = arena_alloc(BB_ARENA, REG_CNT * sizeof(var_t *)); - for (int i = 0; i < REG_CNT; i++) { - var_t *var = REGS[i].var; - - /* Hand over only what the successor is going to read: pinning a - * register it has no use for keeps the value live across the boundary - * for nothing, and the backend then has to materialise it where it - * would otherwise have folded it into its consumer. - */ - succ->entry_regs[i] = - var && var_list_holds(&succ->live_in, var) ? var : NULL; - } -} - -/* Install the register file this block starts with: the predecessor's file when - * it handed one over, an empty one otherwise. Everything handed over has just - * been written back, so nothing is polluted on entry. - */ -void load_entry_regs(basic_block_t *bb) -{ - for (int i = 0; i < REG_CNT; i++) { - var_t *var = bb->entry_regs ? bb->entry_regs[i] : NULL; - - /* A pinned register holds the same variable on every path in. */ - if (pinned_base[i]) { - REGS[i].polluted = 0; - continue; - } - - if (REGS[i].var && REGS[i].var != var) - vreg_clear_phys(REGS[i].var); - REGS[i].var = var; - REGS[i].polluted = 0; - vreg_map_to_phys(var, i); - } -} - -/* The operand of 'OP_push' should not been killed until function called. */ -void extend_liveness(basic_block_t *bb, - const insn_t *insn, - var_t *var, - int offset) -{ - if (check_live_out(bb, var)) - return; - if (insn->idx + offset > var->consumed) - var->consumed = insn->idx + offset; -} - -/* Materialize a wide argument exactly where the ABI consumes it. Moving a pair - * after arbitrary allocation is not generally safe: the source and destination - * pairs can overlap in a cycle. A spill/reload costs a little more in that - * uncommon path, but gives both 32-bit backends an unambiguous pair in the - * required ABI registers. - */ -void prepare_pair_argument(basic_block_t *bb, var_t *var, int low) -{ - int high = low + 1; - int old_low = vreg_get_phys(var); - int old_high = vreg_get_phys_hi(var); - - if (old_low == low && old_high == high && REGS[low].var == var && - pair_high_owner[high] == var) - return; - - if (old_low >= 0 && old_low < REG_CNT && REGS[old_low].var == var) - spill_var(bb, var, old_low); - else - vreg_clear_phys(var); - - if (REGS[low].var) - spill_var(bb, REGS[low].var, low); - if (pair_high_owner[high]) { - var_t *owner = pair_high_owner[high]; - int owner_low = vreg_get_phys(owner); - - if (owner_low >= 0 && owner_low < REG_CNT && - REGS[owner_low].var == owner) - spill_var(bb, owner, owner_low); - else - vreg_clear_phys(owner); - } - - vreg_map_pair_to_phys(var, low, high); - load_var(bb, var, low); -} - -/* Reserve the outgoing stack-argument area @func's calls need. - * - * add_func() reserves one pointer-sized word for each source parameter that - * cannot go in a register, which is enough only while every argument takes one - * ABI word. A long long takes two on a 32-bit target, and may be preceded by a - * padding word, so a call can reach well past that reservation: RISC-V, which - * passes MAX_PARAMS arguments in registers, reserves nothing at all. The words - * abi_lower_call_args() stores above the reservation then overwrote the first - * locals of the caller's frame. Nothing is placed in the frame before register - * allocation, so growing the reservation here moves every slot above it. - */ -void reserve_outgoing_args(func_t *func) -{ - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - int cursor = 0; - - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->opcode != OP_push) { - cursor = 0; - continue; - } - /* Aligning every argument bounds both placement rules. */ - cursor = abi_arg_next(cursor, insn->rs1, true); - if ((cursor - MAX_ARGS_IN_REG) * PTR_SIZE > func->stack_size) - func->stack_size = (cursor - MAX_ARGS_IN_REG) * PTR_SIZE; - } - } -} - -/* The first ABI word of each argument of the call being staged, in push order, - * and the next one OP_push takes. abi_lower_call_args() fills them in once per - * call, since only it sees the callee and so which arguments are variadic. - */ -int call_arg_starts[MAX_PARAMS + 1]; -int call_arg_next; - -/* Load only the low word of the register pair @var into @reg. */ -void load_low_word(basic_block_t *bb, var_t *var, int reg) -{ - int held = vreg_get_phys(var); - ph2_ir_t *ir; - - if (held >= 0 && held < REG_CNT && REGS[held].var == var) { - ir = bb_add_ph2_ir(bb, OP_assign); - ir->src0 = held; - } else if (var->is_const) { - ir = bb_add_ph2_ir(bb, OP_load_constant); - ir->src0 = var->init_val; - } else { - ir = bb_add_ph2_ir(bb, var->is_global ? OP_global_load : OP_load); - ir->src0 = var->offset; - ir->ofs_based_on_stack_top = var->ofs_based_on_stack_top; - ir->size_bytes = 4; - ir->is_volatile = var_is_volatile_object(var); - } - ir->dest = reg; -} - -/* Return whether extra arguments are pushed onto stack. */ -bool abi_lower_call_args(basic_block_t *bb, insn_t *insn) -{ - /* An aggregate-return call has one ABI-only destination in addition to the - * MAX_PARAMS source arguments. - */ - insn_t *pushes[MAX_PARAMS + 1]; - int *starts = call_arg_starts; - int num_of_args = 0; - int cursor = 0; - insn_t *call = insn; - func_t *callee = NULL; - int first_param = 0; - - while (call && call->opcode == OP_push) - call = call->next; - if (call && call->opcode == OP_call) - callee = find_func(call->str); - else if (call && call->opcode == OP_indirect) - callee = get_func_signature(call->rs1); - if (callee && callee->returns_aggregate) - first_param = 1; - - call_arg_next = 0; - while (insn && insn->opcode == OP_push) { - bool variadic; - - if (num_of_args >= MAX_PARAMS + 1) - fatal("Too many call arguments"); - variadic = callee && callee->va_args && - num_of_args - first_param >= callee->num_params; - starts[num_of_args] = abi_arg_start(cursor, insn->rs1, variadic); - pushes[num_of_args++] = insn; - cursor = abi_arg_next(cursor, insn->rs1, variadic); - insn = insn->next; - } - - if (cursor <= MAX_ARGS_IN_REG) - return false; - - for (int i = num_of_args - 1; i >= 0; i--) { - insn = pushes[i]; - - /* A split long long leaves its high word in the first stack slot. Its - * low word is placed with the register arguments. - */ - if (abi_arg_is_split(starts[i], insn->rs1)) { - int scratch = MAX_ARGS_IN_REG - 2; - - prepare_pair_argument(bb, insn->rs1, scratch); - ph2_ir_t *ir = bb_add_ph2_ir(bb, OP_store); - ir->src0 = scratch + 1; - ir->src1 = 0; - ir->size_bytes = 4; - spill_var(bb, insn->rs1, scratch); - continue; - } - - if (starts[i] < MAX_ARGS_IN_REG) - continue; - - if (abi_arg_words(insn->rs1) == 2) { - int scratch = MAX_ARGS_IN_REG - 2; - - prepare_pair_argument(bb, insn->rs1, scratch); - ph2_ir_t *ir = bb_add_ph2_ir(bb, OP_store); - ir->src0 = scratch; - ir->src0_hi = scratch + 1; - ir->src1 = (starts[i] - MAX_ARGS_IN_REG) * PTR_SIZE; - ir->size_bytes = 8; - spill_var(bb, insn->rs1, scratch); - continue; - } - - /* A pinned variable has no slot to load from: its value only ever lives - * in its register, so reading the frame here handed the callee whatever - * the slot happened to hold. - */ - int held = pinned_reg_of(insn->rs1); - - if (held >= 0) { - ph2_ir_t *mv = bb_add_ph2_ir(bb, OP_assign); - mv->src0 = held; - mv->dest = MAX_ARGS_IN_REG - 1; - } else { - load_var(bb, insn->rs1, MAX_ARGS_IN_REG - 1); - } - ph2_ir_t *ir = bb_add_ph2_ir(bb, OP_store); - ir->src0 = MAX_ARGS_IN_REG - 1; - ir->src1 = (starts[i] - MAX_ARGS_IN_REG) * PTR_SIZE; - } - REGS[MAX_ARGS_IN_REG - 1].var = NULL; - return true; -} - -/* The half-open range of instruction indices in @bb over which @var holds a - * value, written into *lo and *hi. - * - * Returns false when @var is not live anywhere in @bb. - * - * A value live on entry starts before the first instruction; one defined here - * starts just after the instruction that writes it, so that a value read by - * that same instruction has already ended. It ends at its last read, or past - * the last instruction when it is live on exit. Where a phi-coalescing - * candidate is live, inverted once per function. - * - * phi_slot_conflicts() and vars_interfere() each ask, for one variable at a - * time, which blocks it is live in, and the only way to answer that from the - * liveness sets is to walk every block of the function and scan that block's - * live_in and live_kill for the variable. A self-compile asks 1.59 million - * times and 97% of the answers are "nowhere in this block", so nearly all of - * that scanning finds nothing. - * - * Turning the sets inside out once per function -- from "which variables does - * this block hold" into "which blocks hold this variable" -- costs one pass - * over the sets and reduces every later query to a walk of the handful of - * blocks the variable really occupies. - * - * Candidacy is marked on the variable itself, reusing the stamp pair - * compute_live_in() left behind: liveness analysis has finished by the time - * register allocation runs, so kill_gen and in_gen are dead, and bumping the - * same liveness_gen counter keeps this pass's stamps distinct from the values - * that analysis wrote. - */ -int phi_cand_head[MAX_LOCALS]; -int phi_cand_tail[MAX_LOCALS]; -int phi_cand_num; -int phi_cand_gen; - -/* The block records, chained per candidate through live_blk_next. The buffer is - * kept between functions and only ever grows, so the whole compile pays for the - * largest function once. - */ -basic_block_t **live_blk_bb; -int *live_blk_next; -int *live_blk_cand; -int *live_blk_lo; -int *live_blk_hi; -char *live_blk_flags; -int live_blk_cap; -int live_blk_num; - -/* Bits of live_blk_flags. */ -#define LIVE_REC_IN 1 -#define LIVE_REC_OUT 2 - -/* Where each candidate is written and last read inside the block being indexed, - * filled by one walk of that block's instructions and read back only for the - * candidates that block records. - */ -int phi_cand_def[MAX_LOCALS]; -int phi_cand_last[MAX_LOCALS]; - -/* False once a function outgrows the candidate table; every query then falls - * back to walking all blocks, which is what this pass did before the index. - */ -bool phi_live_ready; - -/* The candidate number of @var under the current function's stamp, or -1 when - * @var is not one. - */ -int phi_cand_index(const var_t *var) -{ - if (!var) - return -1; - if (var->kill_gen != phi_cand_gen) - return -1; - return var->in_gen; -} - -void phi_cand_add(var_t *var) -{ - if (!var) - return; - if (var->kill_gen == phi_cand_gen) - return; - if (phi_cand_num >= MAX_LOCALS) { - phi_live_ready = false; - return; - } - var->kill_gen = phi_cand_gen; - var->in_gen = phi_cand_num; - phi_cand_head[phi_cand_num] = -1; - phi_cand_tail[phi_cand_num] = -1; - phi_cand_num++; -} - -void live_blk_reserve(int want) -{ - if (want <= live_blk_cap) - return; - - int cap = live_blk_cap; - if (cap < 1024) - cap = 1024; - while (cap < want) - cap = cap << 1; - - live_blk_bb = arena_realloc(GENERAL_ARENA, (char *) live_blk_bb, - live_blk_cap * sizeof(basic_block_t *), - cap * sizeof(basic_block_t *)); - live_blk_next = - arena_realloc(GENERAL_ARENA, (char *) live_blk_next, - live_blk_cap * sizeof(int), cap * sizeof(int)); - live_blk_cand = - arena_realloc(GENERAL_ARENA, (char *) live_blk_cand, - live_blk_cap * sizeof(int), cap * sizeof(int)); - live_blk_lo = arena_realloc(GENERAL_ARENA, (char *) live_blk_lo, - live_blk_cap * sizeof(int), cap * sizeof(int)); - live_blk_hi = arena_realloc(GENERAL_ARENA, (char *) live_blk_hi, - live_blk_cap * sizeof(int), cap * sizeof(int)); - live_blk_flags = - arena_realloc(GENERAL_ARENA, live_blk_flags, live_blk_cap, cap); - live_blk_cap = cap; -} - -/* Record that @var is live somewhere in @bb, returning that block's record for - * @var. Blocks arrive in order, so a variable already recorded for this block - * -- it is in both live_in and live_kill -- is caught by looking at the tail of - * its chain rather than by searching. - */ -int phi_live_note(basic_block_t *bb, var_t *var, int flags) -{ - int idx = phi_cand_index(var); - if (idx < 0) - return -1; - - int tail = phi_cand_tail[idx]; - if (tail >= 0 && live_blk_bb[tail] == bb) { - live_blk_flags[tail] |= flags; - return tail; - } - - live_blk_reserve(live_blk_num + 1); - int rec = live_blk_num++; - live_blk_bb[rec] = bb; - live_blk_next[rec] = -1; - live_blk_cand[rec] = idx; - live_blk_flags[rec] = flags; - if (tail < 0) - phi_cand_head[idx] = rec; - else - live_blk_next[tail] = rec; - phi_cand_tail[idx] = rec; - return rec; -} - -/* Note that @var is live on exit from @bb, when @bb already records it. */ -void phi_live_mark_out(const basic_block_t *bb, var_t *var) -{ - int idx = phi_cand_index(var); - if (idx < 0) - return; - - int tail = phi_cand_tail[idx]; - if (tail >= 0 && live_blk_bb[tail] == bb) - live_blk_flags[tail] |= LIVE_REC_OUT; -} - -/* Collect the variables this pass will ask about -- the destination and operand - * of every unwound phi -- and work out, for each block one of them is live in, - * the range of instruction indices it holds a value over. - * - * The ranges are what phi_slot_conflicts() and vars_interfere() compare, and - * computing them here means one walk of each block for all of its candidates - * instead of a walk per candidate per query. - */ -void phi_live_index_build(func_t *func) -{ - phi_cand_num = 0; - live_blk_num = 0; - phi_live_ready = true; - liveness_gen++; - phi_cand_gen = liveness_gen; - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->opcode != OP_unwound_phi) - continue; - phi_cand_add(insn->rd); - phi_cand_add(insn->rs1); - } - } - - if (!phi_live_ready) - return; - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - int first = live_blk_num; - - for (int i = 0; i < bb->live_in.size; i++) - phi_live_note(bb, bb->live_in.elements[i], LIVE_REC_IN); - for (int i = 0; i < bb->live_kill.size; i++) - phi_live_note(bb, bb->live_kill.elements[i], 0); - - /* Live on exit adds no block: a value live out that this block does not - * write is live in as well, so its record already exists. Marking - * rather than recording keeps that invariant from turning a dataflow - * inconsistency into a range for a block the variable is not live in. - */ - for (int i = 0; i < bb->live_out.size; i++) - phi_live_mark_out(bb, bb->live_out.elements[i]); - - int last = live_blk_num; - if (first == last) - continue; - - for (int r = first; r < last; r++) { - int c = live_blk_cand[r]; - - phi_cand_def[c] = -1; - phi_cand_last[c] = -1; - } - - /* Anything a block's instructions name is in that block's live_in or - * live_kill, so every candidate touched below has a record above and - * its scratch has just been cleared. - */ - int n = 0; - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - int c = phi_cand_index(insn->rs1); - if (c >= 0) - phi_cand_last[c] = n; - c = phi_cand_index(insn->rs2); - if (c >= 0) - phi_cand_last[c] = n; - - /* A select reads a third operand. Leaving it out of the walk ends - * the value's range before the instruction that reads it, and two - * variables live at once then look free to share a slot. - */ - c = phi_cand_index(insn->rs3); - if (c >= 0) - phi_cand_last[c] = n; - c = phi_cand_index(insn->rd); - if (c >= 0 && phi_cand_def[c] < 0) - phi_cand_def[c] = n; - n++; - } - - /* A value live on entry starts before the first instruction; one - * defined here starts just after the instruction that writes it, so a - * value read by that same instruction has already ended. It ends at its - * last read, or past the last instruction when it is live on exit. - */ - for (int r = first; r < last; r++) { - int c = live_blk_cand[r]; - int def = phi_cand_def[c], last_read = phi_cand_last[c]; - int lo, hi; - - if ((live_blk_flags[r] & LIVE_REC_IN) || def < 0) - lo = -1; - else - lo = def + 1; - - if (live_blk_flags[r] & LIVE_REC_OUT) - hi = n; - else if (last_read >= 0) - hi = last_read; - else - hi = lo; - - live_blk_lo[r] = lo; - live_blk_hi[r] = hi; - } - } -} - -/* A walk of the blocks a variable can be live in. Backed by the index when it - * has an answer for the variable, and by every block of the function when it - * does not, so the two produce the same results. - */ -typedef struct { - basic_block_t *bb; - int cur; /* the record just returned, -1 when walking every block */ - int rec; - basic_block_t *all; - bool indexed; -} live_iter_t; - -void live_iter_init(live_iter_t *it, func_t *func, var_t *var) -{ - int idx = -1; - - if (phi_live_ready) - idx = phi_cand_index(var); - - if (idx < 0) { - it->indexed = false; - it->rec = -1; - it->all = func->bbs; - } else { - it->indexed = true; - it->rec = phi_cand_head[idx]; - it->all = NULL; - } - it->bb = NULL; - it->cur = -1; -} - -bool live_iter_next(live_iter_t *it) -{ - if (it->indexed) { - int rec = it->rec; - if (rec < 0) { - it->bb = NULL; - it->cur = -1; - return false; - } - it->bb = live_blk_bb[rec]; - it->cur = rec; - it->rec = live_blk_next[rec]; - return true; - } - - basic_block_t *bb = it->all; - if (!bb) { - it->bb = NULL; - it->cur = -1; - return false; - } - it->bb = bb; - it->all = bb->rpo_next; - return true; -} - -/* Advance *@cursor to @bb's record for the candidate whose chain it walks, or - * past it when that candidate is not live there. The chains and every caller's - * outer walk run in the block order of func->bbs, which is increasing rpo, so a - * cursor only ever moves forward. - */ -bool live_rec_seek(int *cursor, const basic_block_t *bb) -{ - int rec = *cursor; - - while (rec >= 0 && live_blk_bb[rec]->rpo < bb->rpo) - rec = live_blk_next[rec]; - *cursor = rec; - return rec >= 0 && live_blk_bb[rec] == bb; -} - -bool var_range_in_bb(basic_block_t *bb, var_t *var, int *lo, int *hi) -{ - bool live_in = var_list_holds(&bb->live_in, var); - - /* A value the block neither receives nor writes cannot be live anywhere in - * it: any read would have made it live on entry, and anything live on exit - * without being written here is live on entry too. Settling that from the - * liveness sets keeps the walk below off the blocks -- the large majority - * -- where the variable never appears, which is what stops this pass from - * costing a quarter of the compiler. - */ - if (!live_in && !var_check_killed(var, bb)) - return false; - - bool live_out = var_list_holds(&bb->live_out, var); - int def = -1, last_read = -1, n = 0; - - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->rs1 == var || insn->rs2 == var || insn->rs3 == var) - last_read = n; - if (insn->rd == var && def < 0) - def = n; - n++; - } - - if (live_in || def < 0) - *lo = -1; - else - *lo = def + 1; - - if (live_out) - *hi = n; - else if (last_read >= 0) - *hi = last_read; - else - *hi = *lo; - return true; -} - -/* Whether @a and @b are ever live at the same point in @func, and so cannot - * share one stack slot. - */ -bool vars_interfere(func_t *func, var_t *a, var_t *b) -{ - live_iter_t it; - live_iter_init(&it, func, a); - - /* Both chains run in block order, so @b's is walked with a cursor that only - * moves forward rather than searched from the start for each of @a's - * blocks. - */ - int b_idx = phi_live_ready ? phi_cand_index(b) : -1; - int b_cur = b_idx >= 0 ? phi_cand_head[b_idx] : -1; - - while (live_iter_next(&it)) { - basic_block_t *bb = it.bb; - int alo, ahi, blo, bhi; - - if (it.cur >= 0) { - alo = live_blk_lo[it.cur]; - ahi = live_blk_hi[it.cur]; - } else if (!var_range_in_bb(bb, a, &alo, &ahi)) - continue; - - if (b_idx >= 0) { - if (!live_rec_seek(&b_cur, bb)) - continue; - blo = live_blk_lo[b_cur]; - bhi = live_blk_hi[b_cur]; - } else if (!var_range_in_bb(bb, b, &blo, &bhi)) - continue; - - if (alo <= bhi && blo <= ahi) - return true; - } - return false; -} - -/* Give a phi operand the same stack slot as the phi it feeds. - * - * Leaving SSA turns each phi into a store of the operand into the phi's slot, - * one per predecessor. The operand is a value of its own, so it is spilled to a - * slot of its own first, and the phi store then reads it back -- a load and a - * store on every path into the join, and on every iteration of a loop. When the - * two never hold values at the same time they can share one slot: the operand - * is written straight into the phi's slot, and the copy the phi would make is a - * load and store of the same address, which the peephole drops. The variables - * this pass has already placed, so a newcomer can be checked against everything - * sharing the slot it is about to join rather than against one member of it. - */ -var_t *phi_slot_vars[MAX_LOCALS]; -int phi_slot_count; - -void phi_slot_record(var_t *var) -{ - /* Dropping a member silently would leave later joiners checked against only - * part of the group sharing their slot, so overflow has to stop the compile - * rather than quietly produce an unsound placement. - */ - if (phi_slot_count >= MAX_LOCALS) - fatal("Too many coalesced phi slots"); - phi_slot_vars[phi_slot_count++] = var; -} - -/* Scratch lists of the variables sharing one slot, gathered before the block - * walks below so that neither has to filter the whole placement table on every - * step of its own loop. - */ -var_t *phi_group_a[MAX_LOCALS]; -var_t *phi_group_b[MAX_LOCALS]; - -/* One cursor per member of the group being checked, so each member's chain is - * walked once across the whole query instead of being searched per block. - */ -int phi_group_cursor[MAX_LOCALS]; - -/* Whether @v is live at the same time as anything already sharing @offset. */ -bool phi_slot_conflicts(func_t *func, int offset, var_t *v) -{ - int group = 0; - - for (int i = 0; i < phi_slot_count; i++) { - var_t *other = phi_slot_vars[i]; - - if (other != v && other->offset == offset) { - int idx = phi_live_ready ? phi_cand_index(other) : -1; - - phi_group_a[group] = other; - phi_group_cursor[group] = idx >= 0 ? phi_cand_head[idx] : -1; - - /* A member the index cannot answer for is marked with a cursor of - * -2 so the walk below falls back to the scanning form for it. - */ - if (idx < 0) - phi_group_cursor[group] = -2; - group++; - } - } - if (!group) - return false; - - /* Blocks are the outer loop so that @v's range in each is worked out once - * rather than once per variable compared against; @v occupies few blocks, - * and the rest are skipped without looking at the group at all. - */ - live_iter_t it; - live_iter_init(&it, func, v); - - while (live_iter_next(&it)) { - basic_block_t *bb = it.bb; - int vlo, vhi; - - if (it.cur >= 0) { - vlo = live_blk_lo[it.cur]; - vhi = live_blk_hi[it.cur]; - } else if (!var_range_in_bb(bb, v, &vlo, &vhi)) - continue; - - for (int i = 0; i < group; i++) { - int olo, ohi; - - if (phi_group_cursor[i] == -2) { - if (!var_range_in_bb(bb, phi_group_a[i], &olo, &ohi)) - continue; - } else { - if (!live_rec_seek(&phi_group_cursor[i], bb)) - continue; - olo = live_blk_lo[phi_group_cursor[i]]; - ohi = live_blk_hi[phi_group_cursor[i]]; - } - if (vlo <= ohi && olo <= vhi) - return true; - } - } - return false; -} - -bool phi_slot_compatible(var_t *d, var_t *s) -{ - if (!slot_is_private(d) || !slot_is_private(s)) - return false; - if (var_slot_size(d) != var_slot_size(s)) - return false; - if (is_pointer_like(d) != is_pointer_like(s)) - return false; - return d->ofs_based_on_stack_top == s->ofs_based_on_stack_top; -} - -/* Move everything sharing @from onto @to, once no member of one group is ever - * live at the same time as a member of the other. Chained phis reach here as - * two groups that were built independently -- "x = phi(...)" placed with its - * own operands before "y = phi(x, ...)" was seen -- and merging them is what - * puts a whole loop-carried variable in one slot. - */ -bool phi_slot_merge(func_t *func, int to, int from) -{ - if (to == from) - return true; - - /* One pass splits the table into the two groups; the pairwise test then - * walks only those, instead of rescanning every placement to find the - * partner for each candidate. - */ - int nto = 0, nfrom = 0; - for (int i = 0; i < phi_slot_count; i++) { - var_t *var = phi_slot_vars[i]; - - if (var->offset == to) - phi_group_a[nto++] = var; - else if (var->offset == from) - phi_group_b[nfrom++] = var; - } - if (!nfrom) - return true; - - for (int i = 0; i < nto; i++) { - for (int j = 0; j < nfrom; j++) { - if (vars_interfere(func, phi_group_a[i], phi_group_b[j])) - return false; - } - } - - for (int j = 0; j < nfrom; j++) - phi_group_b[j]->offset = to; - return true; -} - -void coalesce_phi_slots(func_t *func) -{ - phi_slot_count = 0; - phi_live_index_build(func); - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->opcode != OP_unwound_phi) - continue; - - var_t *d = insn->rd, *s = insn->rs1; - if (!d || !s || d == s) - continue; - if (!phi_slot_compatible(d, s)) - continue; - - if (d->space_is_allocated && s->space_is_allocated) { - phi_slot_merge(func, d->offset, s->offset); - continue; - } - - /* Exactly one side has a slot by now, or neither does. Chained phis - * are met operand-first, so it may be the operand that is already - * placed; whichever side has the slot is the one joined, and when - * neither does the phi takes a fresh slot for the operand to join. - */ - var_t *have, *want; - if (s->space_is_allocated) { - have = s; - want = d; - } else { - if (!d->space_is_allocated) { - alloc_var_slot(func, d); - phi_slot_record(d); - } - have = d; - want = s; - } - - if (phi_slot_conflicts(func, have->offset, want)) - continue; - want->offset = have->offset; - want->space_is_allocated = true; - phi_slot_record(want); - } - } -} - -/* Fill a lowered binary operation from its SSA instruction. Function bodies and - * global setup share it, so a wide initializer such as the element of "long - * long a[] = {1LL << 40}" names the high registers of its operands. - */ -void fill_binary_ph2_ir(ph2_ir_t *ir, - insn_t *insn, - int src0, - int src1, - int dest) -{ - ir->src0 = src0; - ir->src0_hi = vreg_get_phys_hi(insn->rs1); - ir->src1 = src1; - ir->src1_hi = vreg_get_phys_hi(insn->rs2); - ir->dest = dest; - ir->dest_hi = vreg_get_phys_hi(insn->rd); - - /* Record whether the result is an address, and which operand it came from. - * On LP64 an int-typed result has to wrap at 32 bits, while a pointer must - * keep all 64, and pointer arithmetic has to widen the int index beside the - * address. - */ - set_ptr_flags(ir, insn); - ir->size_bytes = insn->rd->ptr_level ? PTR_SIZE : insn->rd->type->size; - - /* SSA temporaries normally retain their result type, but width is a - * property of the operation as well: a wide operand must not be narrowed - * merely because an intermediate lost its annotation. This includes - * comparisons: their result is int, while CMP must inspect the common - * operand width rather than stale high halves of 32-bit register values. - */ - int left_size = insn->rs1->ptr_level ? PTR_SIZE : insn->rs1->type->size; - int right_size = insn->rs2->ptr_level ? PTR_SIZE : insn->rs2->type->size; - if (left_size > ir->size_bytes) - ir->size_bytes = left_size; - - /* A shift is the exception: its type is that of the promoted left operand - * alone, so "(x ^ y) >> (n & 31)" with a long long n is an int shift, and a - * 64-bit shift of that int register brought zeros in as its sign. - */ - if (right_size > ir->size_bytes && insn->opcode != OP_lshift && - insn->opcode != OP_rshift) - ir->size_bytes = right_size; -} - -/* The unary counterpart of fill_binary_ph2_ir(). */ -void fill_unary_ph2_ir(ph2_ir_t *ir, insn_t *insn, int src0, int dest) -{ - ir->src0 = src0; - ir->src0_hi = vreg_get_phys_hi(insn->rs1); - ir->dest = dest; - ir->dest_hi = vreg_get_phys_hi(insn->rd); - - /* As for OP_branch: the width of the test follows the operand, not the - * result. - */ - ir->src0_is_pointer = is_address_like(insn->rs1); - ir->is_unsigned = is_unsigned_scalar(insn->rd); - ir->src0_is_unsigned = is_unsigned_scalar(insn->rs1); - ir->size_bytes = insn->rd->ptr_level ? PTR_SIZE : insn->rd->type->size; - - /* !x yields an int, but it tests all of a long long operand. */ - if (insn->opcode == OP_log_not && !insn->rs1->ptr_level && - insn->rs1->type->size > ir->size_bytes) - ir->size_bytes = insn->rs1->type->size; -} - -/* Narrow a conversion result in @dest again by its own type. - * - * A byte or short sits in its register extended by its own signedness, which a - * later widening or comparison relies on. OP_cast is a move and OP_sign_ext - * extends by the source alone, so a conversion that changes signedness broke - * that: a signed char -17 became unsigned short 0xffffffef, and cast to - * unsigned char it kept every high bit. A signed destination only needs this - * when it is no wider than its unsigned source; any other value already fits. - * The 64-bit x86 backend redoes the extension itself, the others do not. - */ -void narrow_conversion_result(basic_block_t *bb, insn_t *insn, int dest) -{ - var_t *rd = insn->rd, *rs = insn->rs1; - - if (!rd || !rs || rd->ptr_level || rd->is_func || !rd->type || !rs->type || - is_record_type(rd->type) || is_record_type(rs->type)) - return; - - int size = rd->type->size; - bool rd_unsigned = is_unsigned_scalar(rd); - - if ((size != 1 && size != 2) || rd_unsigned == is_unsigned_scalar(rs)) - return; - if (!rd_unsigned && (rs->ptr_level || rs->type->size < size)) - return; - - ph2_ir_t *ir = bb_add_ph2_ir(bb, OP_trunc); - ir->src0 = dest; - ir->src1 = size; - ir->dest = dest; - ir->is_unsigned = rd_unsigned; - ir->size_bytes = size; -} - -/* Place one global initializer, which has no basic block of its own. */ -void reg_alloc_global(insn_t *global_insn) -{ - ph2_ir_t *ir; - int dest, src0; - - /* Global initializers carry no liveness information, so no operand register - * may be reused as a destination here. - */ - - switch (global_insn->opcode) { - case OP_allocat: - if (global_insn->rd->array_size) { - /* Original scheme: pointer slot + backing region. Cache the base - * offset of the backing region into init_val so later global - * initializers can address elements without loading the pointer. - */ - global_insn->rd->offset = GLOBAL_FUNC->stack_size; - global_insn->rd->space_is_allocated = true; - GLOBAL_FUNC->stack_size += PTR_SIZE; - src0 = GLOBAL_FUNC->stack_size; /* base of backing region */ - - /* Stash base offset for this array variable */ - global_insn->rd->init_val = src0; - - if (global_insn->rd->ptr_level || global_insn->rd->is_func) - GLOBAL_FUNC->stack_size += - align_size(PTR_SIZE * global_insn->rd->array_size); - else { - GLOBAL_FUNC->stack_size += align_size( - global_insn->rd->array_size * global_insn->rd->type->size); - } - - dest = - prepare_dest(GLOBAL_FUNC->bbs, NULL, global_insn->rd, -1, -1); - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, OP_global_address_of); - ir->src0 = src0; - ir->dest = dest; - ir->is_pointer = true; - ir->size_bytes = PTR_SIZE; - spill_var(GLOBAL_FUNC->bbs, global_insn->rd, dest); - } else { - global_insn->rd->offset = GLOBAL_FUNC->stack_size; - global_insn->rd->space_is_allocated = true; - if (global_insn->rd->ptr_level || global_insn->rd->is_func) - GLOBAL_FUNC->stack_size += PTR_SIZE; - else if (global_insn->rd->type != TY_int && - global_insn->rd->type != TY_short && - global_insn->rd->type != TY_char && - global_insn->rd->type != TY_bool) { - GLOBAL_FUNC->stack_size += - align_size(global_insn->rd->type->size); - } else - /* 'char' is aligned to one byte for the convenience */ - GLOBAL_FUNC->stack_size += PTR_SIZE; - } - break; - case OP_load_constant: - case OP_load_data_address: - case OP_load_rodata_address: - dest = prepare_dest(GLOBAL_FUNC->bbs, NULL, global_insn->rd, -1, -1); - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, global_insn->opcode); - ir->src0 = global_insn->rd->init_val; - ir->src1 = global_insn->rd->init_val_hi; - ir->dest = dest; - ir->dest_hi = vreg_get_phys_hi(global_insn->rd); - ir->is_unsigned = is_unsigned_scalar(global_insn->rd); - ir->size_bytes = - global_insn->rd->ptr_level ? PTR_SIZE : global_insn->rd->type->size; - break; - case OP_assign: - if (global_insn->rs1 && global_insn->rs1->is_global_address) { - src0 = - prepare_dest(GLOBAL_FUNC->bbs, NULL, global_insn->rs1, -1, -1); - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, OP_global_address_of); - ir->src0 = global_insn->rs1->offset; - ir->dest = src0; - ir->is_pointer = true; - ir->size_bytes = PTR_SIZE; - } else - src0 = prepare_operand(GLOBAL_FUNC->bbs, global_insn->rs1, -1); - dest = prepare_dest(GLOBAL_FUNC->bbs, NULL, global_insn->rd, src0, -1); - - /* An initializer such as "unsigned long long g = c ? 7U : 1U" has an - * int-sized value; a move would leave the high register unwritten, so - * extend it by its own signedness instead. - */ - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, - var_needs_register_pair(global_insn->rd) && - !var_needs_register_pair(global_insn->rs1) - ? OP_cast - : OP_assign); - ir->src0 = src0; - ir->src0_hi = vreg_get_phys_hi(global_insn->rs1); - ir->src0_is_unsigned = is_unsigned_scalar(global_insn->rs1); - ir->src0_is_pointer = is_address_like(global_insn->rs1); - ir->dest = dest; - ir->dest_hi = vreg_get_phys_hi(global_insn->rd); - spill_var(GLOBAL_FUNC->bbs, global_insn->rd, dest); - /* release the unused constant number in register manually */ - REGS[src0].polluted = 0; - vreg_clear_phys(REGS[src0].var); - REGS[src0].var = NULL; - break; - case OP_address_of: - case OP_global_address_of: - /* A global function-pointer initializer first forms the address of its - * global storage slot. Its address is GP-relative, unlike an address - * formed in a function body. - */ - dest = prepare_dest(GLOBAL_FUNC->bbs, NULL, global_insn->rd, -1, -1); - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, OP_global_address_of); - - /* Global arrays have a pointer slot followed by their backing region; - * an address constant for the array denotes that backing region. - */ - ir->src0 = global_insn->rs1->array_size ? global_insn->rs1->init_val - : global_insn->rs1->offset; - ir->dest = dest; - ir->is_pointer = true; - ir->size_bytes = PTR_SIZE; - if (global_insn->rd->is_global_address) { - global_insn->rd->offset = ir->src0; - global_insn->rd->space_is_allocated = true; - - /* This temporary carries a compile-time GP-relative offset, not a - * stack-resident value. Leaving it mapped in REGS[] lets the next - * global setup instruction spill its address to `sp + offset`; on a - * global array that offset is the backing-region address, so the - * spill overwrites an initialized element. The folded - * OP_add/OP_assign paths rematerialize this address from offset, so - * it has no runtime register lifetime. - */ - REGS[dest].polluted = 0; - vreg_clear_phys(global_insn->rd); - REGS[dest].var = NULL; - } - break; - case OP_add: { - /* Special-case address computation for globals: if rs1 is a global base - * and rs2 is a constant, propagate absolute offset to rd so OP_write - * can fold into OP_global_store. - */ - if (global_insn->rs1 && global_insn->rs1->is_global_address && - global_insn->rs2) { - global_insn->rd->offset = - global_insn->rs1->offset + global_insn->rs2->init_val; - global_insn->rd->is_global_address = true; - global_insn->rd->space_is_allocated = true; - break; - } - if (global_insn->rs1 && global_insn->rs1->is_global && - global_insn->rs2) { - int base_off = global_insn->rs1->offset; - - /* For global arrays, use backing-region base cached in init_val */ - if (global_insn->rs1->array_size > 0) - base_off = global_insn->rs1->init_val; - global_insn->rd->offset = base_off + global_insn->rs2->init_val; - global_insn->rd->space_is_allocated = true; - global_insn->rd->is_global = true; - break; - } - /* Fall through to the ordinary scalar binary lowering below. */ - goto lower_global_binary; - } - case OP_negate: - case OP_bit_not: - case OP_log_not: { - /* Unary wide global constant expressions use the normal phase-2 - * instruction too. Keeping these separate from binary lowering avoids - * preparing a nonexistent right operand. - */ - src0 = prepare_operand(GLOBAL_FUNC->bbs, global_insn->rs1, -1); - dest = prepare_dest(GLOBAL_FUNC->bbs, NULL, global_insn->rd, src0, -1); - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, global_insn->opcode); - fill_unary_ph2_ir(ir, global_insn, src0, dest); - break; - } - case OP_sub: - case OP_mul: - case OP_div: - case OP_mod: - case OP_lshift: - case OP_rshift: - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: - case OP_eq: - case OP_neq: - case OP_lt: - case OP_leq: - case OP_gt: - case OP_geq: - lower_global_binary: { - /* Global scalar initializers may have been parsed as a binary constant - * expression. Use the same phase-2 operation as a function body so wide - * division, comparison, and bitwise expressions do not stop at global - * setup merely because they are not pointer-address arithmetic. - */ - int src1; - src0 = prepare_operand(GLOBAL_FUNC->bbs, global_insn->rs1, -1); - src1 = prepare_operand(GLOBAL_FUNC->bbs, global_insn->rs2, src0); - dest = - prepare_dest(GLOBAL_FUNC->bbs, NULL, global_insn->rd, src0, src1); - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, global_insn->opcode); - fill_binary_ph2_ir(ir, global_insn, src0, src1, dest); - break; - } - case OP_write: { - if (global_insn->rs2 && global_insn->rs2->is_func) { - /* A direct aggregate field address is itself a global object, not a - * pointer value stored in that object. Loading it here turns an - * all-zero callback slot into the destination address. - */ - if (global_insn->rs1 && global_insn->rs1->is_global) { - dest = prepare_dest(GLOBAL_FUNC->bbs, NULL, global_insn->rs1, - -1, -1); - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, OP_global_address_of); - ir->src0 = global_insn->rs1->array_size - ? global_insn->rs1->init_val - : global_insn->rs1->offset; - ir->dest = dest; - ir->is_pointer = true; - ir->size_bytes = PTR_SIZE; - src0 = dest; - } else { - src0 = prepare_operand(GLOBAL_FUNC->bbs, global_insn->rs1, -1); - } - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, OP_address_of_func); - ir->src0 = src0; - ir->func_name = intern_string(global_insn->rs2->var_name); - - /* The GP-relative address exists only to receive this one - * relocation. In particular, an address temporary for a later array - * element must not be spilled: its slot allocation can be the very - * element being initialized, which would overwrite the function - * address with the address itself. - */ - REGS[src0].polluted = 0; - vreg_clear_phys(REGS[src0].var); - REGS[src0].var = NULL; - if (dynlink) { - func_t *target_fn = find_func(ir->func_name); - if (target_fn) - target_fn->is_used = true; - } - break; - } - /* Fold (addr, val) where addr carries GP-relative offset */ - if (global_insn->rs1 && (global_insn->rs1->is_global)) { - int vreg; - - if (global_insn->rs2 && global_insn->rs2->is_global_address) { - vreg = prepare_dest(GLOBAL_FUNC->bbs, NULL, global_insn->rs2, - -1, -1); - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, OP_global_address_of); - ir->src0 = global_insn->rs2->offset; - ir->dest = vreg; - ir->is_pointer = true; - ir->size_bytes = PTR_SIZE; - } else - vreg = prepare_operand(GLOBAL_FUNC->bbs, global_insn->rs2, -1); - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, OP_global_store); - ir->src0 = vreg; - ir->src0_hi = vreg_get_phys_hi(global_insn->rs2); - - /* For array variables used as base, store to the backing region's - * base offset (cached in init_val). - */ - int base_off = global_insn->rs1->offset; - if (global_insn->rs1->array_size > 0) - base_off = global_insn->rs1->init_val; - ir->src1 = base_off; - - /* OP_global_store selects its instruction width from this field. - * Leaving it at zero falls through to an eight-byte store, which - * makes a global int field initializer overwrite its successor. - */ - ir->size_bytes = global_insn->sz; - ir->is_pointer = global_insn->rs2->ptr_level > 0; - ir->is_unsigned = is_unsigned_scalar(global_insn->rs2); - - /* A materialized global address is consumed by this one store. Its - * descriptor offset is GP-relative data, not a spill slot; - * retaining the vreg mapping lets the next initializer spill the - * address at sp + offset and overwrite the source element (for - * example, a pointer row initialized from &values[0][1]). - */ - if (global_insn->rs2 && global_insn->rs2->is_global_address) { - REGS[vreg].polluted = 0; - vreg_clear_phys(global_insn->rs2); - REGS[vreg].var = NULL; - } - break; - } - /* Fallback generic write */ - int src1; - src0 = prepare_operand(GLOBAL_FUNC->bbs, global_insn->rs1, -1); - src1 = prepare_operand(GLOBAL_FUNC->bbs, global_insn->rs2, src0); - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, OP_write); - ir->src0 = src0; - ir->src1 = src1; - ir->src1_hi = vreg_get_phys_hi(global_insn->rs2); - ir->dest = global_insn->sz; - set_ptr_flags(ir, global_insn); - break; - } - case OP_trunc: - case OP_sign_ext: - case OP_cast: - /* A narrowing initializer such as "char g[] = {65, 66}" reaches the - * global block as a conversion, so it has to be lowered here exactly as - * it is inside a function. - */ - src0 = prepare_operand(GLOBAL_FUNC->bbs, global_insn->rs1, -1); - dest = prepare_dest(GLOBAL_FUNC->bbs, NULL, global_insn->rd, src0, -1); - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, global_insn->opcode); - ir->src0 = src0; - ir->src1 = global_insn->sz; - ir->dest = dest; - - /* Widening "long long g[] = {1}" extends into a register pair, and the - * backends select that form and the extension's sign from these fields - * exactly as they do inside a function. - */ - ir->src0_hi = vreg_get_phys_hi(global_insn->rs1); - ir->dest_hi = vreg_get_phys_hi(global_insn->rd); - ir->is_unsigned = is_unsigned_scalar(global_insn->rd); - ir->src0_is_unsigned = is_unsigned_scalar(global_insn->rs1); - if (global_insn->opcode != OP_trunc) - narrow_conversion_result(GLOBAL_FUNC->bbs, global_insn, dest); - break; - default: - printf("Unsupported global operation: %d\n", global_insn->opcode); - fflush(stdout); /* see fatal() */ - abort(); - } -} - -/* Write the value @insn just gave an address-taken scalar to its slot. - * - * A pointer to the variable reads the slot, never the register, and nothing the - * allocator tracks says when that happens: the variable need not be named again - * for a read through the pointer, or a callee handed it, to want this value. A - * global is stored straight after its assignment for the same reason. Records - * and arrays are left alone, since their name holds an address and what a - * pointer reaches is the storage behind it. - */ -void store_addressed_def(basic_block_t *bb, const insn_t *insn) -{ - var_t *var = insn->rd; - - if (!var || !var->address_taken || var->is_global) - return; - if (insn->opcode == OP_allocat) - return; - if (var->array_size || var->has_backing_storage) - return; - if (!var->ptr_level && (!var->type || is_record_type(var->type))) - return; - - int reg = find_in_regs(var); - if (reg < 0 || !REGS[reg].polluted) - return; - store_var(bb, var, reg); -} - -/* Assign registers across one basic block, and emit the phase-2 IR that carries - * the assignment. - */ -void reg_alloc_bb(func_t *func, basic_block_t *bb) -{ - bool handle_abi = false, args_on_stack = false; - bool riscv_indirect_target_staged = false; - - is_pushing_args = false; - abi_args_staged = 0; - int args = 0; - - bb->visited++; - - /* The entry block starts with the incoming arguments already in their - * registers; every other block takes what its predecessor handed over, or - * nothing. - */ - if (bb != func->bbs) - load_entry_regs(bb); - - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - func_t *callee_func; - ph2_ir_t *ir; - int dest, src0, src1; - int sz, clear_reg; - - refresh(bb, insn); - - switch (insn->opcode) { - case OP_unwound_phi: - track_var_use(insn->rs1, insn->idx); - - /* A pinned destination lives in the same register on every path, so - * the copy this phi stands for is a register move rather than a - * write into a slot nothing reads back. - */ - int to = pinned_reg_of(insn->rd); - if (to >= 0) { - src0 = prepare_operand(bb, insn->rs1, -1); - if (src0 != to) { - ir = bb_add_ph2_ir(bb, OP_assign); - ir->src0 = src0; - ir->dest = to; - ir->is_pointer = is_pointer_like(insn->rd); - ir->size_bytes = var_slot_size(insn->rd); - } - REGS[to].var = insn->rd; - REGS[to].polluted = 1; - break; - } - - if (!insn->rd->space_is_allocated) - alloc_var_slot(bb->belong_to, insn->rd); - - /* Sharing a slot with the phi turns the copy into a write of the - * operand into the place it already lives. Only a register the - * block has changed still needs storing -- and reading the slot - * back first, as the general path would, is a load whose value goes - * straight home again. - */ - if (insn->rs1->space_is_allocated && - insn->rs1->offset == insn->rd->offset && - insn->rs1->ofs_based_on_stack_top == - insn->rd->ofs_based_on_stack_top) { - int held = find_in_regs(insn->rs1); - - if (held < 0 || !REGS[held].polluted) - break; /* the slot already holds the value */ - store_var(bb, insn->rs1, held); - break; - } - - src0 = prepare_operand(bb, insn->rs1, -1); - ir = bb_add_ph2_ir(bb, OP_store); - ir->src0 = src0; - - /* The phi's slot is eight bytes for a pair; a store of the low - * register alone left the high word of the previous iteration. - */ - ir->src0_hi = vreg_get_phys_hi(insn->rs1); - ir->src1 = insn->rd->offset; - ir->ofs_based_on_stack_top = insn->rd->ofs_based_on_stack_top; - ir->is_pointer = is_pointer_like(insn->rd); - ir->size_bytes = var_slot_size(insn->rd); - break; - case OP_allocat: - if ((insn->rd->type == TY_void || insn->rd->type == TY_int || - insn->rd->type == TY_short || insn->rd->type == TY_char || - insn->rd->type == TY_bool) && - insn->rd->array_size == 0) - break; - - insn->rd->offset = func->stack_size; - insn->rd->space_is_allocated = true; - func->stack_size += PTR_SIZE; - src0 = func->stack_size; - - if (insn->rd->ptr_level || insn->rd->is_func) - sz = PTR_SIZE; - else { - sz = insn->rd->type->size; - } - - if (insn->rd->array_size) - func->stack_size += align_size(insn->rd->array_size * sz); - else - func->stack_size += align_size(sz); - - if (!insn->rd->is_global && - aggregate_has_function_pointer(insn->rd->type)) { - insn->rd->has_backing_storage = true; - } - - dest = prepare_dest(bb, insn, insn->rd, -1, -1); - ir = bb_add_ph2_ir(bb, OP_address_of); - ir->src0 = src0; - ir->dest = dest; - ir->ofs_based_on_stack_top = insn->rd->ofs_based_on_stack_top; - - /* For arrays, store the base address just like global arrays do */ - if (insn->rd->array_size) - spill_var(bb, insn->rd, dest); - else if (insn->rd->is_func || - (!insn->rd->ptr_level && !insn->rd->type->ptr_level && - is_record_type(insn->rd->type) && - !insn->rd->has_backing_storage)) { - /* OP_allocat's result is the backing address for a callback - * object, not the pointer value stored in that object. Keeping - * it mapped as the object lets a later conservative spill - * overwrite an initialized callback with its own address. A - * record is reached through &record, which names the slot - * itself, so the same spill would overwrite its first member. - */ - REGS[dest].var = NULL; - REGS[dest].polluted = 0; - vreg_clear_phys(insn->rd); - } - break; - case OP_load_constant: - case OP_load_data_address: - case OP_load_rodata_address: - dest = prepare_dest(bb, insn, insn->rd, -1, -1); - ir = bb_add_ph2_ir(bb, insn->opcode); - ir->src0 = insn->rd->init_val; - ir->src1 = insn->rd->init_val_hi; - ir->dest = dest; - ir->dest_hi = vreg_get_phys_hi(insn->rd); - ir->is_unsigned = is_unsigned_scalar(insn->rd); - ir->size_bytes = - insn->rd->ptr_level ? PTR_SIZE : insn->rd->type->size; - - /* store global variable immediately after assignment */ - if (insn->rd->is_global) { - ir = bb_add_ph2_ir(bb, OP_global_store); - ir->src0 = dest; - ir->src0_hi = vreg_get_phys_hi(insn->rd); - ir->src1 = insn->rd->offset; - ir->is_volatile = var_is_volatile_object(insn->rd); - REGS[dest].polluted = 0; - } - - break; - case OP_address_of: - case OP_global_address_of: - /* Mark variable as address-taken, disable constant optimization */ - insn->rs1->address_taken = true; - insn->rs1->is_const = false; - - /* Source-level record parameters arrive as hidden pointers to the - * caller's by-value copy. Their declaration must stay a record for - * parsing and type checks, but taking its address produces the ABI - * pointer rather than allocating a scalar spill slot. - */ - if (insn->rs1->is_aggregate_param) { - int param_idx = -1; - for (int i = 0; i < func->num_params; i++) - if (insn->rs1->base == &func->param_defs[i] || - insn->rs1 == &func->param_defs[i]) { - param_idx = i; - break; - } - if (param_idx < 0) - fatal("Aggregate parameter is not owned by its function"); - - dest = prepare_dest(bb, insn, insn->rd, -1, -1); - - /* A variadic callee reserves the final named aggregate's - * complete ABI footprint in its contiguous argument-save area. - * `va_start` obtains that slot's address, then advances by the - * same rounded extent to the first unnamed argument. - */ - if (func->va_args && param_idx + 1 == func->num_params && - insn->rs1->space_is_allocated) { - ir = bb_add_ph2_ir(bb, OP_address_of); - ir->src0 = insn->rs1->offset; - ir->dest = dest; - ir->ofs_based_on_stack_top = - insn->rs1->ofs_based_on_stack_top; - } else if (insn->rs1->space_is_allocated) { - ir = bb_add_ph2_ir(bb, OP_load); - ir->src0 = insn->rs1->offset; - ir->dest = dest; - ir->ofs_based_on_stack_top = - insn->rs1->ofs_based_on_stack_top; - } else if (abi_param_start(func, param_idx) < MAX_ARGS_IN_REG) { - ir = bb_add_ph2_ir(bb, OP_assign); - ir->src0 = abi_param_start(func, param_idx); - ir->dest = dest; - } else { - ir = bb_add_ph2_ir(bb, OP_load); - ir->src0 = - (abi_param_start(func, param_idx) - MAX_ARGS_IN_REG) * - PTR_SIZE; - ir->dest = dest; - ir->ofs_based_on_stack_top = true; - } - ir->is_pointer = true; - ir->size_bytes = PTR_SIZE; - break; - } - - /* OP_allocat puts a local aggregate's spill slot before its backing - * storage. &aggregate must name the backing storage, not the spill - * slot. - * - * FIXME: This does not support aggregate parameter for now. - */ - bool is_pointer = insn->rs1->ptr_level || - (insn->rs1->type && insn->rs1->type->ptr_level); - if (!insn->rs1->is_global && !is_pointer && - aggregate_has_function_pointer(insn->rs1->type)) { - if (!insn->rs1->has_backing_storage) { - insn->rs1->offset = func->stack_size; - insn->rs1->space_is_allocated = true; - insn->rs1->ofs_based_on_stack_top = false; - func->stack_size += PTR_SIZE; - if (insn->rs1->ptr_level) - sz = PTR_SIZE; - else - sz = insn->rs1->type->size; - if (insn->rs1->array_size) - func->stack_size += - align_size(insn->rs1->array_size * sz); - else - func->stack_size += align_size(sz); - insn->rs1->has_backing_storage = true; - } - - dest = prepare_dest(bb, insn, insn->rd, -1, -1); - ir = bb_add_ph2_ir(bb, OP_address_of); - ir->src0 = insn->rs1->offset + PTR_SIZE; - ir->dest = dest; - ir->ofs_based_on_stack_top = insn->rs1->ofs_based_on_stack_top; - break; - } - - /* make sure variable is on stack */ - if (!insn->rs1->space_is_allocated) { - alloc_var_slot(bb->belong_to, insn->rs1); - - for (int i = 0; i < REG_CNT; i++) - if (REGS[i].var == insn->rs1 && !pinned_base[i]) { - ir = bb_add_ph2_ir(bb, OP_store); - ir->src0 = i; - - /* A pair spills both words: the slot is about to be - * read through the address, not from this register. - */ - ir->src0_hi = vreg_get_phys_hi(insn->rs1); - ir->src1 = insn->rs1->offset; - ir->ofs_based_on_stack_top = - insn->rs1->ofs_based_on_stack_top; - /* Clear stale register tracking */ - REGS[i].var = NULL; - vreg_clear_phys(insn->rs1); - } - } - - dest = prepare_dest(bb, insn, insn->rd, -1, -1); - if (insn->rs1->is_global || insn->opcode == OP_global_address_of) - ir = bb_add_ph2_ir(bb, OP_global_address_of); - else - ir = bb_add_ph2_ir(bb, OP_address_of); - ir->src0 = insn->rs1->offset; - ir->dest = dest; - ir->ofs_based_on_stack_top = insn->rs1->ofs_based_on_stack_top; - break; - case OP_cmov: { - /* A select reads three registers, one more than the allocator's - * avoid arguments can protect, so each is locked as it is placed. - * With all three safe the destination may land anywhere. - */ - int cond, taken, other; - - track_var_use(insn->rs1, insn->idx); - track_var_use(insn->rs2, insn->idx); - track_var_use(insn->rs3, insn->idx); - - reg_locked = 0; - taken = prepare_operand(bb, insn->rs1, -1); - reg_locked = reg_locked | (1 << taken); - other = prepare_operand(bb, insn->rs3, taken); - reg_locked = reg_locked | (1 << other); - cond = prepare_operand(bb, insn->rs2, taken); - reg_locked = reg_locked | (1 << cond); - dest = prepare_dest(bb, insn, insn->rd, taken, other); - - if (dest < 0) { - /* A select needs a fourth register only while all three inputs - * remain live. Save one unpinned input first, then use its - * physical register as the result. The CMOV emitter - * deliberately supports the destination aliasing either arm; it - * tests the condition before overwriting anything, so the - * condition is safe too if it is the only choice. - * - * spill_var() leaves the machine register unchanged, which is - * exactly what the select still needs. It only removes the - * allocator's association, making the value available for the - * result and forcing a later use of the saved input to reload - * its slot. - */ - int reuse = -1; - const int sources[] = {taken, other, cond}; - - for (int i = 0; i < 3; i++) { - int reg = sources[i]; - if (!pinned_base[reg]) { - reuse = reg; - break; - } - } - - /* pin_registers() reserves at most half the file, so one of a - * select's inputs is always reclaimable. - */ - if (reuse < 0) - abort(); - spill_var(bb, REGS[reuse].var, reuse); - dest = prepare_dest(bb, insn, insn->rd, taken, other); - if (dest != reuse) - abort(); - } - reg_locked = 0; - ir = bb_add_ph2_ir(bb, OP_cmov); - ir->src0 = cond; - ir->src1 = taken; - ir->src2 = other; - ir->dest = dest; - - /* As for OP_branch, the recorded width is the condition's: a select - * tests that value, and the arms are moved whole. An int condition - * is decided by its low word alone, whatever the width of the - * result. - */ - ir->size_bytes = - insn->rs2->ptr_level ? PTR_SIZE : insn->rs2->type->size; - ir->src0_is_pointer = is_address_like(insn->rs2); - ir->is_pointer = is_pointer_like(insn->rd); - break; - } - case OP_assign: - /* A copy nothing reads is dropped, unless it copies from or to a - * volatile object. Copying from one is the read a discarded - * expression such as "status;" asks for, and a write to one is a - * side effect however little the program reads it back. - */ - if (insn->rd->consumed == -1 && - !var_is_volatile_object(insn->rs1) && - !var_is_volatile_object(insn->rd)) - break; - - track_var_use(insn->rs1, insn->idx); - src0 = find_in_regs(insn->rs1); - - /* A volatile object whose register matches its storage is read - * again, as prepare_operand() does. - */ - if (src0 > -1 && var_is_volatile_object(insn->rs1) && - !REGS[src0].polluted) - src0 = -1; - - /* If operand is loaded from stack, clear the original slot after - * moving. - */ - if (src0 > -1) - clear_reg = 0; - else { - clear_reg = 1; - src0 = prepare_operand(bb, insn->rs1, -1); - } - dest = prepare_dest(bb, insn, insn->rd, src0, -1); - ir = bb_add_ph2_ir(bb, OP_assign); - ir->src0 = src0; - ir->src0_hi = vreg_get_phys_hi(insn->rs1); - ir->dest = dest; - ir->dest_hi = vreg_get_phys_hi(insn->rd); - ir->is_unsigned = is_unsigned_scalar(insn->rd); - ir->src0_is_unsigned = is_unsigned_scalar(insn->rs1); - ir->src0_is_pointer = is_address_like(insn->rs1); - ir->size_bytes = - insn->rd->ptr_level ? PTR_SIZE : insn->rd->type->size; - - /* store global variable immediately after assignment */ - if (insn->rd->is_global) { - ir = bb_add_ph2_ir(bb, OP_global_store); - ir->src0 = dest; - ir->src0_hi = vreg_get_phys_hi(insn->rd); - ir->src1 = insn->rd->offset; - ir->is_unsigned = is_unsigned_scalar(insn->rd); - ir->size_bytes = var_slot_size(insn->rd); - ir->is_volatile = var_is_volatile_object(insn->rd); - REGS[dest].polluted = 0; - } - - if (clear_reg) { - vreg_clear_phys(REGS[src0].var); - REGS[src0].var = NULL; - } - - /* An assignment creates storage-backed state. It must not retain a - * literal's compile-time cache for later reads. - */ - insn->rd->is_const = false; - - break; - case OP_read: - src0 = prepare_operand(bb, insn->rs1, -1); - dest = prepare_dest(bb, insn, insn->rd, src0, -1); - ir = bb_add_ph2_ir(bb, OP_read); - ir->src0 = src0; - ir->src1 = insn->sz; - ir->dest = dest; - ir->dest_hi = vreg_get_phys_hi(insn->rd); - ir->is_volatile = insn_reads_volatile(insn); - set_ptr_flags(ir, insn); - break; - case OP_write: - if (insn->rs2->is_func) { - src0 = prepare_operand(bb, insn->rs1, -1); - ir = bb_add_ph2_ir(bb, OP_address_of_func); - ir->src0 = src0; - ir->func_name = intern_string(insn->rs2->var_name); - if (dynlink) { - func_t *target_fn = find_func(ir->func_name); - if (target_fn) - target_fn->is_used = true; - } - } else { - /* FIXME: Register content becomes stale after store operation. - * Current workaround causes redundant spilling - need better - * register invalidation strategy. - */ - spill_alive(bb, insn); - src0 = prepare_operand(bb, insn->rs1, -1); - src1 = prepare_operand(bb, insn->rs2, src0); - ir = bb_add_ph2_ir(bb, OP_write); - ir->src0 = src0; - ir->src1 = src1; - ir->src1_hi = vreg_get_phys_hi(insn->rs2); - ir->dest = insn->sz; - set_ptr_flags(ir, insn); - } - break; - case OP_branch: - src0 = prepare_operand(bb, insn->rs1, -1); - - /* Read the high register now: spill_live_out_keep() below forgets - * the mapping of a condition that does not outlive the block. - */ - src1 = vreg_get_phys_hi(insn->rs1); - - /* REGS[src0].var had been set to NULL, but the actual content is - * still holded in the register. - * - * Write every live-out value back but keep it in its register: the - * arm reached by the jump starts with an empty file and loads from - * the slots just written, while the arm that falls through can - * inherit the registers through bb_export_regs(). - */ - spill_live_out_keep(bb); - - ir = bb_add_ph2_ir(bb, OP_branch); - ir->src0 = src0; - ir->src0_hi = src1; - - /* An LP64 backend tests an address over its full width and an int - * over its low word only. - */ - ir->src0_is_pointer = is_address_like(insn->rs1); - ir->src0_is_unsigned = is_unsigned_scalar(insn->rs1); - - /* A long long is tested whole too, which AArch64 selects by the - * width recorded here. - */ - ir->size_bytes = - insn->rs1->ptr_level ? PTR_SIZE : insn->rs1->type->size; - ir->then_bb = bb->then_; - ir->else_bb = bb->else_; - break; - case OP_push: - extend_liveness(bb, insn, insn->rs1, insn->sz); - - if (!is_pushing_args) { - /* RV32 has four ABI argument registers. A call with two aligned - * 64-bit arguments fills all four, leaving no virtual register - * in which OP_indirect may materialize its target. Stage it - * before assigning that file. The RV32 backend keeps this value - * in s2, an otherwise-unused callee-saved register. - */ - if (ELF_MACHINE == 0xf3) { - insn_t *call = insn; - - while (call && call->opcode == OP_push) - call = call->next; - if (call && call->opcode == OP_indirect) { - src0 = prepare_operand(bb, call->rs1, -1); - ir = bb_add_ph2_ir(bb, OP_load_func); - ir->src0 = src0; - riscv_indirect_target_staged = true; - } - } - spill_alive(bb, insn); - is_pushing_args = true; - } - if (!handle_abi) { - args_on_stack = abi_lower_call_args(bb, insn); - handle_abi = true; - } - - args = call_arg_starts[call_arg_next++]; - if (args_on_stack && args >= MAX_ARGS_IN_REG) { - args += abi_arg_words(insn->rs1); - break; - } - - if (abi_arg_words(insn->rs1) == 2) { - if (abi_arg_is_split(args, insn->rs1)) { - /* Nothing is placed after it in a register, so the last one - * takes the low word without being claimed. - */ - load_low_word(bb, insn->rs1, args); - abi_args_staged |= 1 << args; - args += 2; - break; - } - prepare_pair_argument(bb, insn->rs1, args); - abi_args_staged |= 3 << args; - args += 2; - break; - } - - src0 = prepare_operand(bb, insn->rs1, -1); - ir = bb_add_ph2_ir(bb, OP_assign); - ir->src0 = src0; - ir->dest = args; - ir->is_unsigned = is_unsigned_scalar(insn->rs1); - ir->src0_is_unsigned = is_unsigned_scalar(insn->rs1); - REGS[ir->dest].var = insn->rs1; - REGS[ir->dest].polluted = 0; - abi_args_staged |= 1 << args; - args++; - break; - case OP_call: - callee_func = find_func(insn->str); - if (!callee_func->num_params) - spill_alive(bb, insn); - - if (dynlink) - callee_func->is_used = true; - - ir = bb_add_ph2_ir(bb, OP_call); - /* add_insn() interned this when the call was created. */ - ir->func_name = insn->str; - - is_pushing_args = false; - abi_args_staged = 0; - args = 0; - handle_abi = false; - - clobber_caller_saved(); - - break; - case OP_indirect: - if (!args) - spill_alive(bb, insn); - - if (!riscv_indirect_target_staged) { - src0 = prepare_operand(bb, insn->rs1, -1); - ir = bb_add_ph2_ir(bb, OP_load_func); - ir->src0 = src0; - ir->src0_is_unsigned = is_unsigned_scalar(insn->rs1); - } - - bb_add_ph2_ir(bb, OP_indirect); - - is_pushing_args = false; - abi_args_staged = 0; - args = 0; - handle_abi = false; - riscv_indirect_target_staged = false; - - clobber_caller_saved(); - break; - case OP_func_ret: - dest = prepare_dest(bb, insn, insn->rd, -1, -1); - ir = bb_add_ph2_ir(bb, OP_assign); - ir->src0 = 0; - if (var_needs_register_pair(insn->rd)) - ir->src0_hi = 1; - ir->dest = dest; - ir->dest_hi = vreg_get_phys_hi(insn->rd); - ir->is_unsigned = is_unsigned_scalar(insn->rd); - ir->size_bytes = - insn->rd->ptr_level ? PTR_SIZE : insn->rd->type->size; - break; - case OP_return: - if (insn->rs1) - src0 = prepare_operand(bb, insn->rs1, -1); - else - src0 = -1; - - ir = bb_add_ph2_ir(bb, OP_return); - ir->src0 = src0; - ir->src0_hi = vreg_get_phys_hi(insn->rs1); - ir->src0_is_unsigned = is_unsigned_scalar(insn->rs1); - if (insn->rs1) - ir->size_bytes = - insn->rs1->ptr_level ? PTR_SIZE : insn->rs1->type->size; - break; - case OP_add: - case OP_sub: - case OP_mul: - case OP_div: - case OP_mod: - case OP_lshift: - case OP_rshift: - case OP_eq: - case OP_neq: - case OP_gt: - case OP_geq: - case OP_lt: - case OP_leq: - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: - track_var_use(insn->rs1, insn->idx); - track_var_use(insn->rs2, insn->idx); - src0 = prepare_operand(bb, insn->rs1, -1); - src1 = prepare_operand(bb, insn->rs2, src0); - dest = prepare_dest(bb, insn, insn->rd, src0, src1); - ir = bb_add_ph2_ir(bb, insn->opcode); - fill_binary_ph2_ir(ir, insn, src0, src1, dest); - break; - case OP_negate: - case OP_bit_not: - case OP_log_not: - src0 = prepare_operand(bb, insn->rs1, -1); - dest = prepare_dest(bb, insn, insn->rd, src0, -1); - ir = bb_add_ph2_ir(bb, insn->opcode); - fill_unary_ph2_ir(ir, insn, src0, dest); - break; - case OP_trunc: - case OP_sign_ext: - case OP_cast: - src0 = prepare_operand(bb, insn->rs1, -1); - dest = prepare_dest(bb, insn, insn->rd, src0, -1); - ir = bb_add_ph2_ir(bb, insn->opcode); - ir->src1 = insn->sz; - ir->src0 = src0; - ir->src0_hi = vreg_get_phys_hi(insn->rs1); - ir->dest = dest; - ir->dest_hi = vreg_get_phys_hi(insn->rd); - ir->is_unsigned = is_unsigned_scalar(insn->rd); - ir->src0_is_unsigned = is_unsigned_scalar(insn->rs1); - ir->size_bytes = - insn->rd->ptr_level ? PTR_SIZE : insn->rd->type->size; - if (insn->opcode != OP_trunc) - narrow_conversion_result(bb, insn, dest); - break; - default: - printf("Unknown opcode\n"); - fflush(stdout); /* see fatal() */ - abort(); - } - - store_addressed_def(bb, insn); - } - - if (bb->next) { - spill_live_out_keep(bb); - bb_export_regs(bb); - } else if (bb->then_ || bb->else_) { - /* A conditional branch has already written its live-out values back at - * OP_branch; only the handover is left. - */ - bb_export_regs(bb); - } - - if (bb == func->exit) - return; - - /* append jump instruction for the normal block only */ - if (!bb->next) - return; - - if (bb->next == func->exit) - return; - - /* jump to the beginning of loop or over the else block */ - if (bb->next->visited == func->visited || bb->next->rpo != bb->rpo + 1) { - ph2_ir_t *ir = bb_add_ph2_ir(bb, OP_jump); - ir->next_bb = bb->next; - } -} - -void reg_alloc(void) -{ - /* TODO: Add proper .bss and .data section support for uninitialized / - * initialized globals - */ - for (insn_t *global_insn = GLOBAL_FUNC->bbs->insn_list.head; global_insn; - global_insn = global_insn->next) { - reg_alloc_global(global_insn); - } - - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - func->visited++; - fatal_function_context = func->return_def.var_name; - - if (!strcmp(func->return_def.var_name, "main")) - MAIN_BB = func->bbs; - - for (int i = 0; i < REG_CNT; i++) { - REGS[i].var = NULL; - pair_high_owner[i] = NULL; - } - - slot_var_count = 0; - reserve_outgoing_args(func); - coalesce_phi_slots(func); - pin_registers(func); - - /* Set arguments available. The register file is indexed by ABI word, - * whereas param_defs is indexed by source parameter. - */ - int args_in_reg = 0; - if (func->returns_aggregate) { - REGS[0].var = var_subscript0(&func->sret_def); - REGS[0].polluted = 1; - } - for (int i = 0; i < func->num_params; i++) { - int word = abi_param_start(func, i); - - if (word + abi_arg_words(&func->param_defs[i]) > MAX_ARGS_IN_REG) - break; - REGS[word].var = var_subscript0(&func->param_defs[i]); - REGS[word].polluted = 1; - if (abi_arg_words(&func->param_defs[i]) == 2) - vreg_map_pair_to_phys(REGS[word].var, word, word + 1); - args_in_reg++; - } - - /* Move a pinned parameter out of the argument register it arrived in - * before anything else claims that register. Pinned registers are - * handed out from the top of the file and pin_registers() refuses any - * that an argument occupies, so this never overwrites another one. - */ - for (int i = 0; i < args_in_reg; i++) { - var_t *param = var_subscript0(&func->param_defs[i]); - int home = pinned_reg_of(param); - int word = abi_param_start(func, i); - - if (home < 0) - continue; - - ph2_ir_t *mv = bb_add_ph2_ir(func->bbs, OP_assign); - mv->src0 = word; - mv->dest = home; - REGS[home].var = param; - REGS[home].polluted = 0; - REGS[word].var = NULL; - REGS[word].polluted = 0; - } - - /* variadic function implementation */ - if (func->va_args) { - /* When encountering a variadic function, allocate space for all - * arguments on the local stack to ensure their addresses are - * contiguous. - * - * On a 32-bit target a long long argument takes two words, after a - * padding word when it would start at an odd one, so a call can - * pass twice as many words as it has arguments: save that many. - * va_arg rounds its cursor to an even word by rounding the address - * to eight bytes, so the first word must sit at an address that is - * a multiple of eight too. - */ - int va_words = PTR_SIZE < 8 ? 2 * MAX_PARAMS : MAX_PARAMS; - - /* A stack word is fetched through the register after the argument - * registers. RISC-V passes as many words in registers as the file - * holds, so it borrows the last argument register instead, which - * has been saved by then. - */ - int scratch = - MAX_ARGS_IN_REG < REG_CNT ? MAX_ARGS_IN_REG : REG_CNT - 1; - - if (PTR_SIZE < 8) - func->stack_size = ALIGN_UP(func->stack_size, 8); - for (int i = 0; i < va_words; i++) { - ph2_ir_t *ir; - int src0 = i; - - if (i >= MAX_ARGS_IN_REG) { - /* Callee should access caller's stack to obtain the extra - * arguments. - */ - ir = bb_add_ph2_ir(func->bbs, OP_load); - ir->dest = scratch; - ir->src0 = (i - MAX_ARGS_IN_REG) * PTR_SIZE; - ir->ofs_based_on_stack_top = true; - src0 = scratch; - } - - /* Slot i saves ABI word i. The parameter homed there is found - * by its ABI position, since a hidden aggregate-return pointer - * takes word 0 and shifts every named parameter up. - */ - int param_idx = -1; - for (int q = 0; q < func->num_params; q++) { - if (abi_param_start(func, q) == i) { - param_idx = q; - break; - } - } - - if (param_idx >= 0) { - var_t *param = var_subscript0(&func->param_defs[param_idx]); - param->offset = func->stack_size; - param->space_is_allocated = true; - } - - ir = bb_add_ph2_ir(func->bbs, OP_store); - ir->src0 = src0; - ir->src1 = func->stack_size; - func->stack_size += PTR_SIZE; - if (param_idx >= 0) { - var_t *param = var_subscript0(&func->param_defs[param_idx]); - - if (param_idx + 1 == func->num_params && - param->is_aggregate_param) { - int footprint = ALIGN_UP(param->type->size, PTR_SIZE); - - /* The first word was just saved above. Reserve the - * remaining words before spilling unnamed arguments. - */ - func->stack_size += footprint - PTR_SIZE; - } - } - } - - /* A parameter that arrived in the borrowed register now lives only - * in its save slot. - */ - if (va_words > MAX_ARGS_IN_REG && scratch < MAX_ARGS_IN_REG) { - var_t *owner = pair_high_owner[scratch]; - - if (owner) { - int low = vreg_get_phys(owner); - - if (low >= 0 && low < REG_CNT && REGS[low].var == owner) { - REGS[low].var = NULL; - REGS[low].polluted = 0; - } - vreg_clear_phys(owner); - } else if (REGS[scratch].var) { - vreg_clear_phys(REGS[scratch].var); - REGS[scratch].var = NULL; - REGS[scratch].polluted = 0; - } - } - } else { - /* If the number of function arguments is fixed, the extra arguments - * are directly placed in the caller's stack space instead of the - * callee's. - * - * +----------> +---------------+ - * | | local vars | - * | +---------------+ - * | | extra arg 4 | - * | +---------------+ <-- sp + stack_size + 12 - * caller's space | extra arg 3 | - * | +---------------+ <-- sp + stack_size + 8 - * | | extra arg 2 | - * | +---------------+ <-- sp + stack_size + 4 - * | | extra arg 1 | - * +----------> +---------------+ <-- sp + stack_size - * | | local vars | - * | +---------------+ <-- sp + 16 - * callee's space | Next callee's | - * | | additional | - * | | arguments | - * +----------> +---------------+ <-- sp - * - * Note that: - * - For the Arm architecture, extra arg1 ~ argX correspond to - * arg5 ~ arg(X + 4). - * - For the RISC-V architecture, extra arg1 ~ argX correspond to - * arg9 ~ arg(X + 8). - * - * If any instruction use one of these additional arguments, it - * inherits 'offset' and 'ofs_based_on_stack_top'. When calling - * cfg_flatten(), the operand's offset will be recalculated by - * adding the function's stack size. - */ - for (int i = 0; i < func->num_params; i++) { - var_t *param = var_subscript0(&func->param_defs[i]); - int word = abi_param_start(func, i); - - if (abi_arg_is_split(word, param)) { - /* A long long split between a7 and the first stack slot - * gets a slot of its own: store the low word, then fetch - * the high word through a7, which holds nothing else. - */ - ph2_ir_t *ir; - - if (!param->space_is_allocated) - alloc_var_slot(func, param); - ir = bb_add_ph2_ir(func->bbs, OP_store); - ir->src0 = word; - ir->src1 = param->offset; - ir->size_bytes = 4; - ir = bb_add_ph2_ir(func->bbs, OP_load); - ir->dest = word; - ir->src0 = 0; - ir->ofs_based_on_stack_top = true; - ir->size_bytes = 4; - ir = bb_add_ph2_ir(func->bbs, OP_store); - ir->src0 = word; - ir->src1 = param->offset + 4; - ir->size_bytes = 4; - continue; - } - if (word < MAX_ARGS_IN_REG) - continue; - param->offset = (word - MAX_ARGS_IN_REG) * PTR_SIZE; - param->space_is_allocated = true; - param->ofs_based_on_stack_top = true; - } - } - - /* A volatile parameter lives in its slot, so write it there on entry. - * Its register then matches the slot, and every read reloads it. - */ - for (int i = 0; i < args_in_reg; i++) { - var_t *param = var_subscript0(&func->param_defs[i]); - int word = abi_param_start(func, i); - - if (var_is_volatile_object(param) && REGS[word].var == param) - store_var(func->bbs, param, word); - } - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - reg_alloc_bb(func, bb); - } - - /* handle implicit return */ - for (int i = 0; i < func->exit->prev_idx; i++) { - basic_block_t *bb = func->exit->prev[i].bb; - if (!bb) - continue; - - if (func->return_def.type != TY_void) - continue; - - if (bb->insn_list.tail) - if (bb->insn_list.tail->opcode == OP_return) - continue; - - ph2_ir_t *ir = bb_add_ph2_ir(bb, OP_return); - ir->src0 = -1; - } - - slot_scan(func); - collapse_slot_roundtrip(func); - dead_store_elim(func); - } -} - -/* Name the high register of a 32-bit target's register pair after its low one, - * as "%x2:%x3", when @reg is one. - */ -void dump_pair_high(int reg) -{ - if (reg >= 0) - printf(":%%x%c", reg + 48); -} - -void dump_ph2_ir(void) -{ - for (int i = 0; i < ph2_ir_idx; i++) { - ph2_ir_t *ph2_ir = PH2_IR_FLATTEN[i]; - - const int rd = ph2_ir->dest + 48; - const int rs1 = ph2_ir->src0 + 48; - const int rs2 = ph2_ir->src1 + 48; - - switch (ph2_ir->op) { - case OP_define: - printf("%s:", ph2_ir->func_name); - break; - case OP_allocat: - continue; - case OP_assign: - printf("\t%%x%c = %%x%c", rd, rs1); - break; - case OP_cmov: - printf("\t%%x%c = %%x%c if %%x%c else %%x%c", rd, rs2, rs1, - ph2_ir->src2 + 48); - break; - case OP_load_constant: - printf("\tli %%x%c, $%d", rd, ph2_ir->src0); - break; - case OP_load_data_address: - printf("\t%%x%c = .data(%d)", rd, ph2_ir->src0); - break; - case OP_load_rodata_address: - printf("\t%%x%c = .rodata(%d)", rd, ph2_ir->src0); - break; - case OP_address_of: - printf("\t%%x%c = %%sp + %d", rd, ph2_ir->src0); - break; - case OP_global_address_of: - printf("\t%%x%c = %%gp + %d", rd, ph2_ir->src0); - break; - case OP_branch: - printf("\tbr %%x%c", rs1); - break; - case OP_jump: - printf("\tj %s", ph2_ir->func_name ? ph2_ir->func_name : ""); - break; - case OP_call: - printf("\tcall @%s", ph2_ir->func_name); - break; - case OP_return: - if (ph2_ir->src0 == -1) - printf("\tret"); - else - printf("\tret %%x%c", rs1); - break; - case OP_load: - printf("\tload %%x%c", rd); - dump_pair_high(ph2_ir->dest_hi); - printf(", %d(sp)", ph2_ir->src0); - break; - case OP_store: - printf("\tstore %%x%c", rs1); - dump_pair_high(ph2_ir->src0_hi); - printf(", %d(sp)", ph2_ir->src1); - break; - case OP_global_load: - printf("\tload %%x%c", rd); - dump_pair_high(ph2_ir->dest_hi); - printf(", %d(gp)", ph2_ir->src0); - break; - case OP_global_store: - printf("\tstore %%x%c", rs1); - dump_pair_high(ph2_ir->src0_hi); - printf(", %d(gp)", ph2_ir->src1); - break; - case OP_read: - printf("\t%%x%c = (%%x%c)", rd, rs1); - break; - case OP_write: - printf("\t(%%x%c) = %%x%c", rs1, rs2); - break; - case OP_address_of_func: - printf("\t(%%x%c) = @%s", rs1, ph2_ir->func_name); - break; - case OP_load_func: - printf("\tload %%t0, %d(sp)", ph2_ir->src0); - break; - case OP_global_load_func: - printf("\tload %%t0, %d(gp)", ph2_ir->src0); - break; - case OP_indirect: - printf("\tindirect call @(%%t0)"); - break; - case OP_negate: - printf("\tneg %%x%c, %%x%c", rd, rs1); - break; - case OP_add: - printf("\t%%x%c = add %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_sub: - printf("\t%%x%c = sub %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_mul: - printf("\t%%x%c = mul %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_div: - printf("\t%%x%c = div %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_mod: - printf("\t%%x%c = mod %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_eq: - printf("\t%%x%c = eq %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_neq: - printf("\t%%x%c = neq %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_gt: - printf("\t%%x%c = gt %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_lt: - printf("\t%%x%c = lt %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_geq: - printf("\t%%x%c = geq %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_leq: - printf("\t%%x%c = leq %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_bit_and: - printf("\t%%x%c = and %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_bit_or: - printf("\t%%x%c = or %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_bit_not: - printf("\t%%x%c = not %%x%c", rd, rs1); - break; - case OP_bit_xor: - printf("\t%%x%c = xor %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_log_not: - printf("\t%%x%c = not %%x%c", rd, rs1); - break; - case OP_rshift: - printf("\t%%x%c = rshift %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_lshift: - printf("\t%%x%c = lshift %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_trunc: - printf("\t%%x%c = trunc %%x%c, %d", rd, rs1, ph2_ir->src1); - break; - case OP_sign_ext: - printf("\t%%x%c = sign_ext %%x%c, %d", rd, rs1, ph2_ir->src1); - break; - case OP_cast: - printf("\t%%x%c = cast %%x%c", rd, rs1); - break; - default: - break; - } - printf("\n"); - } -} diff --git a/src/riscv-codegen.c b/src/riscv-codegen.c index a41a61d4..ccbc2d47 100644 --- a/src/riscv-codegen.c +++ b/src/riscv-codegen.c @@ -12,385 +12,366 @@ #define RV32_ALIGNMENT 16 -/* Explanation: registers preservation/restoration - * - * The following table illustrates which registers are caller-saved or - * callee-saved: - * +-----------+--------+ - * | Register | Saver | - * | ABI Name | | - * +-----------+--------+ - * | zero, | | - * | gp, | (None) | - * | tp | | - * +-----------+--------+ - * | ra | Caller | - * +-----------+--------+ - * | sp | Callee | - * +-----------+--------+ - * | a0 - a7 | Caller | - * +-----------+--------+ - * | s0 - s11 | Callee | - * +-----------+--------+ - * | t0 - t6 | Caller | - * +-----------+--------+ - * - * - ra and sp: are properly handled by the code generator. - * - * - a0 - a7: are implicitly handled in register allocation phase. - * - * Register allocation use 8 virtual registers (vreg0 - vreg7) to - * allocate registers from IR. When encountering a function call, - * all virtual registers are spilled properly before the call, and - * their contents will also be restored if they are used after the - * call. - * - * Since vreg0 - vreg7 map to a0 - a7 directly, these physical - * registers are also preserved/restored naturally whan a function - * is invoked or returned. - * - * - s0 and s1: handling depends on the linking mode. - * - * Static linking: - * For s0 register, it is used at the program entry point under - * static linking. The program entry point will use s0 to store - * the value of sp register and obtain 'argc' and 'argv' via s0 - * before calling the main function. Then, the content of s0 is - * no longer used after calling the main function, so its - * preservation and restoration can also be ignored. - * - * s1 is not used under static linking, so it is not necessary to - * be handled. - * - * Dynamic linking: - * Both registers are used to preserve 'argc' and 'argv' and pass - * them to the main function. Therefore, the code generator ensures - * that proper instructions are generated to preserve/restore - * their contents via stack. - * - * - s2 - s11: are not necessary to be preserved or restored. - * - * The current code generator does not use s2 - s11 to generate - * instructions, so these registers are not needed to be processed. - * - * - t0 - t6: are not necessary to be handled. - * - * These registers are used to store certain temporary values; - * however, the code generator ensures that these values do not - * persist across function calls. +/* How a branch or jump is encoded, kept in its src2. The long forms reach any + * target; rv_relax_branches() picks a shorter one once the layout shows the + * target in range or right behind the instruction. */ +enum { + RV_BRANCH_LONG, /* lui/addi then, beqz +8, jr; lui/addi else, jr */ + RV_BRANCH_THEN, /* bnez then, falling into else */ + RV_BRANCH_ELSE, /* beqz else, falling into then */ + RV_BRANCH_THEN_JAL, /* bnez then; jal else */ + RV_BRANCH_SKIP_THEN, /* beqz +8; jal then, falling into else */ + RV_BRANCH_SKIP_ELSE, /* bnez +8; jal else, falling into then */ +}; -void update_elf_offset(ph2_ir_t *ph2_ir) +enum { + RV_JUMP_LONG, /* auipc/jalr */ + RV_JUMP_JAL, + RV_JUMP_FALL, /* the target follows */ +}; + +/* a0-a7 hold virtual registers 0-7; s3-s11 hold the remaining allocatable + * registers and are saved when used. s2 stages indirect callees, while s0/s1 + * belong to startup argument handling. The backend does not use other saved + * registers; ra and sp are handled by the function prologue and epilogue. + */ + +/* The machine register for IR register @ir_reg. The first eight are the + * argument registers a0-a7; the rest are s3-s11, which a callee preserves, so a + * value pinned there survives a call. s0 and s1 belong to the entry code and s2 + * holds an indirect call's target. + */ +int rv_reg_of(int ir_reg) { - switch (ph2_ir->op) { - case OP_load_constant: - if (ph2_ir->src0 < -2048 || ph2_ir->src0 > 2047) - elf_offset += 8; - else - elf_offset += 4; - if (ph2_ir->dest_hi >= 0) { - if (ph2_ir->src1 < -2048 || ph2_ir->src1 > 2047) - elf_offset += 8; - else - elf_offset += 4; - } - return; - case OP_address_of: - case OP_global_address_of: - if (ph2_ir->src0 < -2048 || ph2_ir->src0 > 2047) - elf_offset += 12; - else - elf_offset += 4; - return; - case OP_assign: - elf_offset += 4; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->dest_hi != ph2_ir->src0_hi) - elf_offset += 4; - /* Exchanging the two registers of a pair takes a third move. */ - if (ph2_ir->dest_hi >= 0 && ph2_ir->dest == ph2_ir->src0_hi && - ph2_ir->dest_hi == ph2_ir->src0) - elf_offset += 4; - return; - case OP_load: - case OP_global_load: - if (ph2_ir->src0 < -2048 || ph2_ir->src0 > 2047) - if (ph2_ir->dest_hi >= 0) - elf_offset += 20; - else - elf_offset += 16; - else if (ph2_ir->dest_hi >= 0) - elf_offset += 8; - else - elf_offset += 4; - return; - case OP_store: - case OP_global_store: - if (ph2_ir->src1 < -2048 || ph2_ir->src1 > 2047) - if (ph2_ir->src0_hi >= 0) - elf_offset += 20; - else - elf_offset += 16; - else if (ph2_ir->src0_hi >= 0) - elf_offset += 8; - else - elf_offset += 4; - return; - case OP_read: - elf_offset += ph2_ir->dest_hi >= 0 ? 8 : 4; - return; - case OP_write: - elf_offset += ph2_ir->src1_hi >= 0 ? 8 : 4; - return; - case OP_call: - /* A call through the PLT may be farther than a JAL reaches. */ - elf_offset += dynlink && !find_func(ph2_ir->func_name)->bbs ? 8 : 4; - return; - case OP_jump: - case OP_load_func: - case OP_indirect: - case OP_lshift: - case OP_rshift: - elf_offset += ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 ? 52 : 4; - return; - case OP_gt: - case OP_lt: - elf_offset += ph2_ir->src0_hi >= 0 && ph2_ir->src1_hi >= 0 ? 28 : 4; - return; - case OP_negate: - elf_offset += 4; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) - elf_offset += 12; - return; - case OP_add: - case OP_sub: - elf_offset += 4; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - elf_offset += 12; - return; - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: - elf_offset += 4; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - elf_offset += 4; - return; - case OP_bit_not: - elf_offset += 4; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) - elf_offset += 4; - return; - case OP_mul: - if (hard_mul_div) { - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - elf_offset += 24; - else - elf_offset += 4; - } else if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - elf_offset += 100; - else - elf_offset += 52; - return; - case OP_div: - case OP_mod: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) + return ir_reg < 8 ? ir_reg + __a0 : __s3 + ir_reg - 8; +} - /* Four copied input words, a quotient pair, a remainder pair, and a - * fixed 64-round restoring loop. The modulo form copies the final - * remainder out after the loop. Signed operands add magnitude - * conversion and a sign-restoration sequence. - */ - elf_offset += ph2_ir->src0_is_unsigned || ph2_ir->src1_is_unsigned - ? (ph2_ir->op == OP_mod ? 168 : 160) - : 244; - else if (hard_mul_div) - elf_offset += 4; - else - elf_offset += 116; - return; - case OP_load_data_address: - case OP_load_rodata_address: - case OP_geq: - case OP_leq: - elf_offset += ph2_ir->src0_hi >= 0 && ph2_ir->src1_hi >= 0 ? 32 : 8; - return; - case OP_log_not: - elf_offset += ph2_ir->src0_hi >= 0 ? 12 : 8; - return; - case OP_address_of_func: - elf_offset += 12; - return; - case OP_eq: - elf_offset += ph2_ir->src0_hi >= 0 && ph2_ir->src1_hi >= 0 ? 20 : 12; - return; - case OP_neq: - elf_offset += ph2_ir->src0_hi >= 0 && ph2_ir->src1_hi >= 0 ? 16 : 8; - return; - case OP_branch: - elf_offset += ph2_ir->src0_hi >= 0 ? 24 : 20; - return; - case OP_return: - elf_offset += 24; - if (ph2_ir->src0_hi >= 0) - elf_offset += 4; - return; - case OP_trunc: - /* A byte and a short each need a shift pair to keep the sign. */ - if (ph2_ir->src1 == 1 || ph2_ir->src1 == 2) - elf_offset += 8; - else - elf_offset += 4; - return; - case OP_sign_ext: { - /* Decode source size from upper 16 bits */ - int source_size = (ph2_ir->src1 >> 16) & 0xFFFF; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi < 0) { - /* Mirror the emitter: a zero-extended byte is one ANDI, a - * sign-extended byte and either halfword take a shift pair, and a - * word is one move; the high word adds one more instruction. The - * byte cases were swapped, so a (long long) cast of an unsigned - * char moved every later branch target by four bytes. - */ - bool zero_ext = ph2_ir->src0_is_unsigned || ph2_ir->src0_is_pointer; - elf_offset += - source_size == 2 || (source_size == 1 && !zero_ext) ? 12 : 8; - return; +/* Bit 9 of rv_saved_regs(): the function stages an indirect call's target in + * s2. + */ +#define RV_SAVED_S2 (1 << 9) + +/* The callee-saved registers @func names, and so must save on entry and restore + * on return: s3-s11 as a mask in bits 0-8, and RV_SAVED_S2. A mask rather than + * the highest one used, because the lowerer stages stack arguments and indirect + * callees through the last of them, s11, which would otherwise drag every lane + * below it into the frame. + */ +int rv_saved_regs(func_t *func) +{ + int saved = func_used_reg_mask(func) >> 8; + + if (func_has_op(func, OP_load_func, OP_load_func)) + saved |= RV_SAVED_S2; + return saved; +} + +/* How many words rv_saved_regs()'s @saved occupies in the frame. */ +int rv_saved_words(int saved) +{ + int words = !!(saved & RV_SAVED_S2); + for (int i = 0; i < CALLEE_SAVED_REGS; i++) + words += (saved >> i) & 1; + return words; +} + +/* The frame a function allocates: its locals, ra, and the saved registers, kept + * 16-byte aligned. + */ +int rv_frame_bytes(int stack_size, int saved) +{ + return ALIGN_UP(stack_size + 4 + rv_saved_words(saved) * 4, RV32_ALIGNMENT); +} + +/* The register saved in frame word @i: the s3-s11 in @saved, then s2. */ +int rv_saved_reg(int saved, int i) +{ + for (int reg = 0; reg < CALLEE_SAVED_REGS; reg++) + if (((saved >> reg) & 1) && !i--) + return __s3 + reg; + return __s2; +} + +void update_elf_offset(vir_mop_t *mop) +{ + if (mop->op != OP_allocat) + elf_offset += mop_count_words(mop) * 4; +} + +static mop_frame_layout_t rv_frame_layout(func_t *func, vir_mop_t *define) +{ + int saved = rv_saved_regs(func); + + define->src2 = saved; + return (mop_frame_layout_t) {rv_frame_bytes(func->stack_size, saved), + mop_count_words(define) * 4, saved, true}; +} + +/* Where the first function is placed, past the startup code and globals. */ +static int rv_functions_start; + +static int rv_insn_bytes(vir_mop_t *ir) +{ + return ir->op == OP_allocat ? 0 : mop_count_words(ir) * 4; +} + +/* The bytes of each function's prologue and each of its instructions, in layout + * order. Only branches and jumps change size as their forms shrink, so the rest + * are measured once. + */ +static int *rv_sizes; + +/* Place every function again with the branch forms chosen so far, in the order + * mop_flatten_functions() placed them, and return where code ends. + */ +static int rv_layout_functions(void) +{ + int offset = rv_functions_start, define = 0, k = 0; + bool first = !rv_sizes; + + if (first) { + /* One spare entry: shecc's own malloc() returns NULL for zero bytes, + * which a unit with no function bodies would otherwise ask for. + */ + rv_sizes = malloc((mop_flat_count + 1) * sizeof(int)); + if (!rv_sizes) + fatal("Out of memory"); + } + for (func_t *func = FUNC_LIST.head; func; func = func->next) { + if (!func->bbs) + continue; + if (first) { + while (MOP_FLAT[define]->op != OP_define) + define++; + rv_sizes[k] = mop_count_words(MOP_FLAT[define++]) * 4; + } + int prologue = rv_sizes[k++]; + for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { + bb->elf_offset = offset; + if (bb == func->bbs) + offset += prologue; + for (vir_mop_t *ir = bb->mops.head; ir; ir = ir->next) { + if (first || ir->op == OP_jump || ir->op == OP_branch) + rv_sizes[k] = rv_insn_bytes(ir); + offset += rv_sizes[k++]; + } } - if (source_size == 2) - elf_offset += 8; /* short extension: 2 instructions */ - else - elf_offset += 12; /* byte extension: 3 instructions */ - return; } - case OP_cast: - elf_offset += - ph2_ir->dest_hi >= 0 && - (ph2_ir->src0_hi < 0 || ph2_ir->dest_hi != ph2_ir->src0_hi) - ? 8 - : 4; - return; - default: - fatal("Unknown opcode"); + return offset; +} + +static bool rv_branch_reaches(int distance) +{ + return distance >= -4096 && distance <= 4094; +} + +static bool rv_jal_reaches(int distance) +{ + return distance >= -1048576 && distance <= 1048574; +} + +/* Choose the shortest form each branch and jump has in the current layout. A + * target "right behind" an instruction starts where it ends; shrinking code + * keeps that true and only brings other targets closer, so a form chosen here + * stays valid in every later layout. + * + * Returns whether any form changed. + */ +static bool rv_choose_branch_forms(void) +{ + bool changed = false; + int k = 0; + + for (func_t *func = FUNC_LIST.head; func; func = func->next) { + if (!func->bbs) + continue; + int prologue = rv_sizes[k++]; + for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { + int offset = bb->elf_offset + (bb == func->bbs ? prologue : 0); + for (vir_mop_t *ir = bb->mops.head; ir; ir = ir->next) { + int size = rv_sizes[k++], end = offset + size; + int form = ir->src2, words = 0; + + if (ir->op == OP_jump && ir->next_bb) { + int target = ir->next_bb->elf_offset; + if (target == end) + form = RV_JUMP_FALL; + else if (rv_jal_reaches(target - offset)) + form = RV_JUMP_JAL; + words = form == RV_JUMP_FALL ? 0 + : form == RV_JUMP_JAL ? 1 + : 2; + } else if (ir->op == OP_branch) { + int q = offset + (ir->src0_hi >= 0 ? 4 : 0); + int then_at = ir->then_bb->elf_offset; + int else_at = ir->else_bb->elf_offset; + if (else_at == end && rv_branch_reaches(then_at - q)) + form = RV_BRANCH_THEN; + else if (then_at == end && rv_branch_reaches(else_at - q)) + form = RV_BRANCH_ELSE; + else if (rv_branch_reaches(then_at - q) && + rv_jal_reaches(else_at - q - 4)) + form = RV_BRANCH_THEN_JAL; + else if (else_at == end && rv_jal_reaches(then_at - q - 4)) + form = RV_BRANCH_SKIP_THEN; + else if (then_at == end && rv_jal_reaches(else_at - q - 4)) + form = RV_BRANCH_SKIP_ELSE; + words = (q - offset) / 4 + + (form == RV_BRANCH_LONG ? 7 + : form == RV_BRANCH_THEN || form == RV_BRANCH_ELSE + ? 1 + : 2); + } + if (form != ir->src2 && words * 4 < size) { + ir->src2 = form; + changed = true; + } + offset = end; + } + } } + return changed; +} + +/* Lay the functions out until the branch forms settle. */ +static void rv_relax_branches(void) +{ + do + elf_offset = rv_layout_functions(); + while (rv_choose_branch_forms()); + free(rv_sizes); + rv_sizes = NULL; } void cfg_flatten(void) { func_t *func; + mop_flat_prepare(false); if (dynlink) { - /* When using dynamic linking, 25 instructions are generated at the + /* When using dynamic linking, 26 instructions are generated at the * program entry point to perform the following operations: * - prepare arguments and call __libc_start_main() * - preserve a0 ('argc'), a1 ('argv') and sp. * - allocate and clear a global stack, then jump to global init * function. */ - elf_offset = 100; + elf_offset = 104; } else { /* Under static linking, "__syscall" must be generated to allow the * program to invoke system calls. * - * "__syscall" consists of 9 instructions, preceded by 6 initial - * instructions. Consequently, the elf offset for "__syscall" is is 24 - * bytes, and the offset for the subsequent function (GLOBAL_FUNC) is 60 - * bytes. + * "__syscall" consists of 9 instructions, preceded by 7 initial + * instructions because the global-init call is an AUIPC/JALR pair. + * Consequently, "__syscall" starts at byte 28 and the subsequent + * function (GLOBAL_FUNC) starts at byte 64. */ func = find_func("__syscall"); - func->bbs->elf_offset = 24; - elf_offset = 60; + func->bbs->elf_offset = 28; + elf_offset = 64; } GLOBAL_FUNC->bbs->elf_offset = elf_offset; - for (ph2_ir_t *ph2_ir = GLOBAL_FUNC->bbs->ph2_ir_list.head; ph2_ir; - ph2_ir = ph2_ir->next) { - update_elf_offset(ph2_ir); - } - - /* prepare 'argc' and 'argv', then proceed to 'main' function */ - if (dynlink) - elf_offset += 44; - else - elf_offset += 24; + mop_visit_globals(update_elf_offset); - for (func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; + /* Prepare argc/argv and reach main through a range-independent call. + * code_generate() emits this only when there is a main, so a translation + * unit without one must not be charged for it either: every function after + * would be placed 28 bytes from where it is emitted. + */ + if (MAIN_BB) + elf_offset += dynlink ? 48 : 28; - /* reserve stack */ - ph2_ir_t *flatten_ir = add_ph2_ir(OP_define); - flatten_ir->src0 = func->stack_size; - flatten_ir->func_name = intern_string(func->return_def.var_name); + rv_functions_start = elf_offset; + mop_flatten_functions(rv_frame_layout, update_elf_offset); + rv_relax_branches(); +} - /* Except for local variables, it must allocate additional space to - * preserve the content of ra at each function entry point. - * - * 'stack_size' doesn't include the additional space, so an extra number - * '4' is added to 'stack_size'. - */ - int stack_top_ofs = ALIGN_UP(func->stack_size + 4, RV32_ALIGNMENT); +void emit(int code) +{ + if (mop_count_only) { + if (mop_counted_words < INT_MAX) + mop_counted_words++; + } else + elf_write_int(elf_code, code); +} - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - bb->elf_offset = elf_offset; +static void emit_rv32_split_address(bool pc_relative, + rv_reg rd, + rv_reg base, + int offset) +{ + int hi = rv_hi(offset), lo = rv_lo(offset); - if (bb == func->bbs) { - /* save ra, sp */ - elf_offset += 16; - } + emit(pc_relative ? __auipc(base, hi) : __lui(base, hi)); + emit(pc_relative ? __jalr(rd, base, lo) : __addi(rd, base, lo)); +} - for (ph2_ir_t *insn = bb->ph2_ir_list.head; insn; - insn = insn->next) { - if (insn->ofs_based_on_stack_top) { - switch (insn->op) { - case OP_load: - case OP_address_of: - insn->src0 = insn->src0 + stack_top_ofs; - break; - case OP_store: - insn->src1 = insn->src1 + stack_top_ofs; - break; - default: - /* Ignore opcodes with the ofs_based_on_stack_top flag - * set since only the three opcodes above needs to - * access a variable's address. - */ - break; - } - } - flatten_ir = add_existed_ph2_ir(insn); +static rv_reg rv_memory_base(rv_reg base, int offset, int pair, int *disp) +{ + *disp = offset; + if (offset >= -2048 && offset <= 2047 - pair * 4) + return base; + emit_rv32_split_address(false, __t0, __t0, offset); + emit(__add(__t0, base, __t0)); + *disp = 0; + return __t0; +} - if (insn->op == OP_return) { - /* restore sp */ - flatten_ir->src1 = bb->belong_to->stack_size; - } +static void rv_load(rv_reg dest, + rv_reg base, + int offset, + int size, + bool is_unsigned) +{ + if (size == 1) + emit(is_unsigned ? __lbu(dest, base, offset) + : __lb(dest, base, offset)); + else if (size == 2) + emit(is_unsigned ? __lhu(dest, base, offset) + : __lh(dest, base, offset)); + else + emit(__lw(dest, base, offset)); +} - update_elf_offset(flatten_ir); - } - } - } +static void rv_store(rv_reg src, rv_reg base, int offset, int size) +{ + if (size == 1) + emit(__sb(src, base, offset)); + else if (size == 2) + emit(__sh(src, base, offset)); + else + emit(__sw(src, base, offset)); } -void emit(int code) +static int riscv_bitwise_insn(opcode_t op, + rv_reg dest, + rv_reg left, + rv_reg right) { - elf_write_int(elf_code, code); + switch (op) { + case OP_bit_and: + return __and(dest, left, right); + case OP_bit_or: + return __or(dest, left, right); + case OP_bit_xor: + return __xor(dest, left, right); + default: + fatal("unsupported RISC-V bitwise operation"); + } + return 0; } -void emit_ph2_ir(ph2_ir_t *ph2_ir) +void emit_mop(vir_mop_t *mop) { - func_t *func; - int rd = ph2_ir->dest + 10; - int rs1 = ph2_ir->src0 + 10; - int rs2 = ph2_ir->src1 + 10; - int rd_hi = ph2_ir->dest_hi + 10; - int rs1_hi = ph2_ir->src0_hi + 10; - int rs2_hi = ph2_ir->src1_hi + 10; + int rd = rv_reg_of(mop->dest); + int rs1 = rv_reg_of(mop->src0); + int rs2 = rv_reg_of(mop->src1); + int rd_hi = rv_reg_of(mop->dest_hi); + int rs1_hi = rv_reg_of(mop->src0_hi); + int rs2_hi = rv_reg_of(mop->src1_hi); int ofs; /* Prepare the variables to reuse the same code for the instruction sequence @@ -401,42 +382,46 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) */ rv_reg interm, divisor_mask = __t1; - switch (ph2_ir->op) { + switch (mop->op) { case OP_define: - fatal_function_context = ph2_ir->func_name; - ofs = ALIGN_UP(ph2_ir->src0 + 4, RV32_ALIGNMENT); - emit(__sw(__ra, __sp, -4)); - emit(__lui(__t0, rv_hi(ofs))); - emit(__addi(__t0, __t0, rv_lo(ofs))); + fatal_function_context = mop->func_name; + ofs = rv_frame_bytes(mop->src0, mop->src2); + + /* Lower sp before saving below the frame's top: there is no red zone, + * and a signal delivered in between would overwrite anything stored + * below sp. t1 holds the old sp to address the saves from. + */ + emit_rv32_split_address(false, __t0, __t0, ofs); + emit(__addi(__t1, __sp, 0)); emit(__sub(__sp, __sp, __t0)); + emit(__sw(__ra, __t1, -4)); + for (int i = 0; i < rv_saved_words(mop->src2); i++) + emit(__sw(rv_saved_reg(mop->src2, i), __t1, -8 - i * 4)); return; case OP_load_constant: - if (ph2_ir->src0 < -2048 || ph2_ir->src0 > 2047) { - emit(__lui(rd, rv_hi(ph2_ir->src0))); - emit(__addi(rd, rd, rv_lo(ph2_ir->src0))); + if (mop->src0 < -2048 || mop->src0 > 2047) { + emit_rv32_split_address(false, rd, rd, mop->src0); } else - emit(__addi(rd, __zero, ph2_ir->src0)); - if (ph2_ir->dest_hi >= 0) { - if (ph2_ir->src1 < -2048 || ph2_ir->src1 > 2047) { - emit(__lui(rd_hi, rv_hi(ph2_ir->src1))); - emit(__addi(rd_hi, rd_hi, rv_lo(ph2_ir->src1))); + emit(__addi(rd, __zero, mop->src0)); + if (mop->dest_hi >= 0) { + if (mop->src1 < -2048 || mop->src1 > 2047) { + emit_rv32_split_address(false, rd_hi, rd_hi, mop->src1); } else - emit(__addi(rd_hi, __zero, ph2_ir->src1)); + emit(__addi(rd_hi, __zero, mop->src1)); } return; case OP_address_of: case OP_global_address_of: - interm = ph2_ir->op == OP_address_of ? __sp : __gp; - if (ph2_ir->src0 < -2048 || ph2_ir->src0 > 2047) { - emit(__lui(__t0, rv_hi(ph2_ir->src0))); - emit(__addi(__t0, __t0, rv_lo(ph2_ir->src0))); + interm = mop->op == OP_address_of ? __sp : __gp; + if (mop->src0 < -2048 || mop->src0 > 2047) { + emit_rv32_split_address(false, __t0, __t0, mop->src0); emit(__add(rd, interm, __t0)); } else - emit(__addi(rd, interm, ph2_ir->src0)); + emit(__addi(rd, interm, mop->src0)); return; case OP_assign: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && rd == rs1_hi) { + if (MOP_PAIR_DEST_SRC0(mop) && rd == rs1_hi) { /* A pair moved one register over, as a slot round trip collapsed * into a move can leave it: the low destination is the high source, * so move the high word first, through t0 when the two registers @@ -453,182 +438,142 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) return; } emit(__addi(rd, rs1, 0)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->dest_hi != ph2_ir->src0_hi) + if (MOP_PAIR_DEST_SRC0(mop) && mop->dest_hi != mop->src0_hi) emit(__addi(rd_hi, rs1_hi, 0)); return; case OP_load: case OP_global_load: - interm = ph2_ir->op == OP_load ? __sp : __gp; - if (ph2_ir->dest_hi >= 0) { - if (ph2_ir->src0 < -2048 || ph2_ir->src0 > 2047) { - emit(__lui(__t0, rv_hi(ph2_ir->src0))); - emit(__addi(__t0, __t0, rv_lo(ph2_ir->src0))); - emit(__add(__t0, interm, __t0)); - emit(__lw(rd, __t0, 0)); - emit(__lw(rd_hi, __t0, 4)); - } else { - emit(__lw(rd, interm, ph2_ir->src0)); - emit(__lw(rd_hi, interm, ph2_ir->src0 + 4)); - } + interm = mop->op == OP_load ? __sp : __gp; + if (mop->dest_hi >= 0) { + int disp; + rv_reg base = rv_memory_base(interm, mop->src0, 1, &disp); + + rv_load(rd, base, disp, 4, true); + rv_load(rd_hi, base, disp + 4, 4, true); return; } - if (ph2_ir->src0 < -2048 || ph2_ir->src0 > 2047) { - emit(__lui(__t0, rv_hi(ph2_ir->src0))); - emit(__addi(__t0, __t0, rv_lo(ph2_ir->src0))); - emit(__add(__t0, interm, __t0)); - if (ph2_ir->size_bytes == 1) - emit(ph2_ir->is_unsigned ? __lbu(rd, __t0, 0) - : __lb(rd, __t0, 0)); - else if (ph2_ir->size_bytes == 2) - emit(ph2_ir->is_unsigned ? __lhu(rd, __t0, 0) - : __lh(rd, __t0, 0)); - else - emit(__lw(rd, __t0, 0)); - } else if (ph2_ir->size_bytes == 1) - emit(ph2_ir->is_unsigned ? __lbu(rd, interm, ph2_ir->src0) - : __lb(rd, interm, ph2_ir->src0)); - else if (ph2_ir->size_bytes == 2) - emit(ph2_ir->is_unsigned ? __lhu(rd, interm, ph2_ir->src0) - : __lh(rd, interm, ph2_ir->src0)); - else - emit(__lw(rd, interm, ph2_ir->src0)); + { + int disp; + rv_reg base = rv_memory_base(interm, mop->src0, 0, &disp); + + rv_load(rd, base, disp, mop->size_bytes, mop->is_unsigned); + } return; case OP_store: case OP_global_store: - interm = ph2_ir->op == OP_store ? __sp : __gp; - if (ph2_ir->src0_hi >= 0) { - if (ph2_ir->src1 < -2048 || ph2_ir->src1 > 2047) { - emit(__lui(__t0, rv_hi(ph2_ir->src1))); - emit(__addi(__t0, __t0, rv_lo(ph2_ir->src1))); - emit(__add(__t0, interm, __t0)); - emit(__sw(rs1, __t0, 0)); - emit(__sw(rs1_hi, __t0, 4)); - } else { - emit(__sw(rs1, interm, ph2_ir->src1)); - emit(__sw(rs1_hi, interm, ph2_ir->src1 + 4)); - } + interm = mop->op == OP_store ? __sp : __gp; + if (mop->src0_hi >= 0) { + int disp; + rv_reg base = rv_memory_base(interm, mop->src1, 1, &disp); + + rv_store(rs1, base, disp, 4); + rv_store(rs1_hi, base, disp + 4, 4); return; } - if (ph2_ir->src1 < -2048 || ph2_ir->src1 > 2047) { - emit(__lui(__t0, rv_hi(ph2_ir->src1))); - emit(__addi(__t0, __t0, rv_lo(ph2_ir->src1))); - emit(__add(__t0, interm, __t0)); - if (ph2_ir->size_bytes == 1) - emit(__sb(rs1, __t0, 0)); - else if (ph2_ir->size_bytes == 2) - emit(__sh(rs1, __t0, 0)); - else - emit(__sw(rs1, __t0, 0)); - } else if (ph2_ir->size_bytes == 1) - emit(__sb(rs1, interm, ph2_ir->src1)); - else if (ph2_ir->size_bytes == 2) - emit(__sh(rs1, interm, ph2_ir->src1)); - else - emit(__sw(rs1, interm, ph2_ir->src1)); + { + int disp; + rv_reg base = rv_memory_base(interm, mop->src1, 0, &disp); + + rv_store(rs1, base, disp, mop->size_bytes); + } return; case OP_read: - if (ph2_ir->dest_hi >= 0) { - /* A pair read through a pointer. When the low destination is the - * address register itself, fetch the high word first so the address - * survives until both words are loaded. - */ - if (ph2_ir->src1 != 8) + if (mop->dest_hi >= 0) { + if (mop->src1 != 8) fatal("unsupported RISC-V pair load width"); - if (rd == rs1) { - emit(__lw(rd_hi, rs1, 4)); - emit(__lw(rd, rs1, 0)); - } else { - emit(__lw(rd, rs1, 0)); - emit(__lw(rd_hi, rs1, 4)); - } +#define RV_WORD_LOAD(dst, base, offset) emit(__lw(dst, base, offset)) + EMIT_PAIR_LOAD(RV_WORD_LOAD, rd, rd_hi, rs1); +#undef RV_WORD_LOAD return; } - if (ph2_ir->src1 == 1) - emit(ph2_ir->is_unsigned ? __lbu(rd, rs1, 0) : __lb(rd, rs1, 0)); - else if (ph2_ir->src1 == 2) - emit(ph2_ir->is_unsigned ? __lhu(rd, rs1, 0) : __lh(rd, rs1, 0)); - else if (ph2_ir->src1 == 4) - emit(__lw(rd, rs1, 0)); - else + if (mop->src1 != 1 && mop->src1 != 2 && mop->src1 != 4) fatal("unsupported RISC-V load width"); + rv_load(rd, rs1, 0, mop->src1, mop->is_unsigned); return; case OP_write: - if (ph2_ir->src1_hi >= 0) { - if (ph2_ir->dest != 8) + if (mop->src1_hi >= 0) { + if (mop->dest != 8) fatal("unsupported RISC-V pair store width"); emit(__sw(rs2, rs1, 0)); emit(__sw(rs2_hi, rs1, 4)); return; } - if (ph2_ir->dest == 1) - emit(__sb(rs2, rs1, 0)); - else if (ph2_ir->dest == 2) - emit(__sh(rs2, rs1, 0)); - else if (ph2_ir->dest == 4) - emit(__sw(rs2, rs1, 0)); - else + if (mop->dest != 1 && mop->dest != 2 && mop->dest != 4) fatal("unsupported RISC-V store width"); + rv_store(rs2, rs1, 0, mop->dest); return; case OP_branch: - ofs = elf_code_start + ph2_ir->then_bb->elf_offset; - emit(__lui(__t0, rv_hi(ofs))); - emit(__addi(__t0, __t0, rv_lo(ofs))); + if (mop->src2 != RV_BRANCH_LONG) { + int form = mop->src2; + + if (mop->src0_hi >= 0) { + emit(__or(__t1, rs1, rs1_hi)); + rs1 = __t1; + } + if (mop_count_only) { + emit(0); + if (form >= RV_BRANCH_THEN_JAL) + emit(0); + return; + } + int here = elf_code->size; + int then_ofs = mop->then_bb->elf_offset - here; + int else_ofs = mop->else_bb->elf_offset - here; + if (form == RV_BRANCH_THEN || form == RV_BRANCH_THEN_JAL) + emit(__bne(rs1, __zero, then_ofs)); + else if (form == RV_BRANCH_ELSE) + emit(__beq(rs1, __zero, else_ofs)); + else if (form == RV_BRANCH_SKIP_THEN) { + emit(__beq(rs1, __zero, 8)); + emit(__jal(__zero, then_ofs - 4)); + } else + emit(__bne(rs1, __zero, 8)); + if (form == RV_BRANCH_THEN_JAL || form == RV_BRANCH_SKIP_ELSE) + emit(__jal(__zero, else_ofs - 4)); + return; + } + ofs = elf_code_start + mop->then_bb->elf_offset; + emit_rv32_split_address(false, __t0, __t0, ofs); /* A pair is true when either word is nonzero. */ - if (ph2_ir->src0_hi >= 0) { + if (mop->src0_hi >= 0) { emit(__or(__t1, rs1, rs1_hi)); rs1 = __t1; } emit(__beq(rs1, __zero, 8)); emit(__jalr(__zero, __t0, 0)); - emit(__jal(__zero, ph2_ir->else_bb->elf_offset - elf_code->size)); + ofs = elf_code_start + mop->else_bb->elf_offset; + emit_rv32_split_address(false, __t0, __t0, ofs); + emit(__jalr(__zero, __t0, 0)); return; case OP_jump: - emit(__jal(__zero, ph2_ir->next_bb->elf_offset - elf_code->size)); - return; - case OP_call: - func = find_func(ph2_ir->func_name); - if (func->bbs) - ofs = func->bbs->elf_offset - elf_code->size; - else if (dynlink) { - /* The PLT follows the code and read-only data, so a large image - * puts it beyond the 1 MiB a JAL reaches. Call it PC-relative - * through AUIPC and JALR, which reach anywhere. - */ - ofs = (dynamic_sections.elf_plt_start + func->plt_offset) - - (elf_code_start + elf_code->size); - emit(__auipc(__ra, rv_hi(ofs))); - emit(__jalr(__ra, __ra, rv_lo(ofs))); + if (mop->src2 == RV_JUMP_FALL) + return; + ofs = mop->next_bb->elf_offset - elf_code->size; + if (mop->src2 == RV_JUMP_JAL) { + emit(mop_count_only ? 0 : __jal(__zero, ofs)); return; - } else { - printf("The '%s' function is not implemented\n", ph2_ir->func_name); - fflush(stdout); /* see fatal() */ - abort(); } - emit(__jal(__ra, ofs)); + emit_rv32_split_address(true, __zero, __t0, ofs); + return; + case OP_call: + /* The PLT follows the code and read-only data, so a large image puts it + * beyond the 1 MiB a JAL reaches. Call it PC-relative through AUIPC and + * JALR, which reach anywhere, as for a defined function. + */ + ofs = function_entry_address(mop->func_name) - + (elf_code_start + elf_code->size); + emit_rv32_split_address(true, __ra, __ra, ofs); return; case OP_load_data_address: - emit(__lui(rd, rv_hi(elf_data_start + ph2_ir->src0))); - emit(__addi(rd, rd, rv_lo(elf_data_start + ph2_ir->src0))); + emit_rv32_split_address(false, rd, rd, elf_data_start + mop->src0); return; case OP_load_rodata_address: - emit(__lui(rd, rv_hi(elf_rodata_start + ph2_ir->src0))); - emit(__addi(rd, rd, rv_lo(elf_rodata_start + ph2_ir->src0))); + emit_rv32_split_address(false, rd, rd, elf_rodata_start + mop->src0); return; case OP_address_of_func: - func = find_func(ph2_ir->func_name); - if (func->bbs) - ofs = elf_code_start + func->bbs->elf_offset; - else if (dynlink) - ofs = dynamic_sections.elf_plt_start + func->plt_offset; - else { - printf("The '%s' function is not implemented\n", ph2_ir->func_name); - fflush(stdout); /* see fatal() */ - abort(); - } - emit(__lui(__t0, rv_hi(ofs))); - emit(__addi(__t0, __t0, rv_lo(ofs))); + ofs = function_entry_address(mop->func_name); + emit_rv32_split_address(false, __t0, __t0, ofs); emit(__sw(__t0, rs1, 0)); return; case OP_load_func: @@ -638,63 +583,68 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) emit(__jalr(__ra, __s2, 0)); return; case OP_return: - if (ph2_ir->src0 == -1) + if (mop->src0 == -1) emit(__addi(__zero, __zero, 0)); else emit(__addi(__a0, rs1, 0)); - if (ph2_ir->src0_hi >= 0) + if (mop->src0_hi >= 0) emit(__addi(__a1, rs1_hi, 0)); - ofs = ALIGN_UP(ph2_ir->src1 + 4, RV32_ALIGNMENT); - emit(__lui(__t0, rv_hi(ofs))); - emit(__addi(__t0, __t0, rv_lo(ofs))); - emit(__add(__sp, __sp, __t0)); - emit(__lw(__ra, __sp, -4)); + ofs = rv_frame_bytes(mop->src1, mop->src2); + /* Reload the saved registers while sp still covers them. */ + emit_rv32_split_address(false, __t0, __t0, ofs); + emit(__add(__t1, __sp, __t0)); + emit(__lw(__ra, __t1, -4)); + for (int i = 0; i < rv_saved_words(mop->src2); i++) + emit(__lw(rv_saved_reg(mop->src2, i), __t1, -8 - i * 4)); + emit(__addi(__sp, __t1, 0)); emit(__jalr(__zero, __ra, 0)); return; case OP_add: - emit(__add(rd, rs1, rs2)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) { - emit(__sltu(__t0, rd, rs1)); + if (MOP_PAIR_BINARY(mop)) { + /* Preserve the original low words until carry is known, including + * an in-place destination and x + x using the same register pair. + */ + emit(__add(__t0, rs1, rs2)); + emit(__sltu(__t1, __t0, rs1)); emit(__add(rd_hi, rs1_hi, rs2_hi)); - emit(__add(rd_hi, rd_hi, __t0)); - } + emit(__add(rd_hi, rd_hi, __t1)); + emit(__addi(rd, __t0, 0)); + } else + emit(__add(rd, rs1, rs2)); return; case OP_sub: - emit(__sub(rd, rs1, rs2)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) { + if (MOP_PAIR_BINARY(mop)) emit(__sltu(__t0, rs1, rs2)); + emit(__sub(rd, rs1, rs2)); + if (MOP_PAIR_BINARY(mop)) { emit(__sub(rd_hi, rs1_hi, rs2_hi)); emit(__sub(rd_hi, rd_hi, __t0)); } return; case OP_mul: if (hard_mul_div) { - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) { - emit(__mul(rd, rs1, rs2)); - emit(__mulhu(rd_hi, rs1, rs2)); + if (MOP_PAIR_BINARY(mop)) { emit(__mul(__t0, rs1, rs2_hi)); emit(__mul(__t1, rs1_hi, rs2)); - emit(__add(rd_hi, rd_hi, __t0)); + emit(__mulhu(__t2, rs1, rs2)); + emit(__mul(rd, rs1, rs2)); + emit(__add(rd_hi, __t2, __t0)); emit(__add(rd_hi, rd_hi, __t1)); } else emit(__mul(rd, rs1, rs2)); } else { - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) { + if (MOP_PAIR_BINARY(mop)) { /* Multiply two 64-bit pairs modulo 2^64 without the M * extension. Keep the product in the destination pair, so t0-t3 * can hold the shifting multiplicand and multiplier; the loop * ends once both multiplier words are zero. */ - emit(__addi(rd, __zero, 0)); - emit(__addi(rd_hi, __zero, 0)); emit(__addi(__t0, rs1, 0)); emit(__addi(__t1, rs1_hi, 0)); emit(__addi(__t2, rs2, 0)); emit(__addi(__t3, rs2_hi, 0)); + emit(__addi(rd, __zero, 0)); + emit(__addi(rd_hi, __zero, 0)); emit(__andi(__t4, __t2, 1)); emit(__sub(__t4, __zero, __t4)); emit(__and(__t5, __t0, __t4)); @@ -733,10 +683,8 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) return; case OP_div: case OP_mod: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) { - bool is_unsigned = - ph2_ir->src0_is_unsigned || ph2_ir->src1_is_unsigned; + if (MOP_PAIR_BINARY(mop)) { + bool is_unsigned = mop->src0_is_unsigned || mop->src1_is_unsigned; /* 64-bit restoring division. t0:t1 is the remainder, t2:t3 is the * shifting dividend, t4:t5 is the divisor, and rd:rd_hi accumulates @@ -813,7 +761,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) emit(__sw(__t6, __sp, 4)); emit(__bne(__t6, __zero, -108)); - if (ph2_ir->op == OP_mod) { + if (mop->op == OP_mod) { emit(__addi(rd, __t0, 0)); emit(__addi(rd_hi, __t1, 0)); } @@ -822,7 +770,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) * follows the dividend's sign. */ emit(__lw(__t6, __sp, 8)); - if (ph2_ir->op == OP_div) { + if (mop->op == OP_div) { emit(__lw(__t0, __sp, 12)); emit(__xor(__t6, __t6, __t0)); } @@ -837,19 +785,19 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) return; } if (hard_mul_div) { - if (ph2_ir->op == OP_div) - emit(ph2_ir->src0_is_unsigned || ph2_ir->src1_is_unsigned + if (mop->op == OP_div) + emit(mop->src0_is_unsigned || mop->src1_is_unsigned ? __divu(rd, rs1, rs2) : __div(rd, rs1, rs2)); else - emit(ph2_ir->src0_is_unsigned || ph2_ir->src1_is_unsigned + emit(mop->src0_is_unsigned || mop->src1_is_unsigned ? __modu(rd, rs1, rs2) : __mod(rd, rs1, rs2)); return; } interm = __t0; /* div/mod emulation */ - if (ph2_ir->op == OP_mod) { + if (mop->op == OP_mod) { /* If the requested operation is modulo, the result will be stored * in __t2. The sign of the divisor is irrelevant for determining * the result's sign. @@ -864,7 +812,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) */ emit(__addi(__t2, rs1, 0)); emit(__addi(__t3, rs2, 0)); - if (ph2_ir->src0_is_unsigned || ph2_ir->src1_is_unsigned) { + if (mop->src0_is_unsigned || mop->src1_is_unsigned) { emit(__addi(__t0, __zero, 0)); emit(__addi(__t2, __t2, 0)); emit(__addi(__t2, __t2, 0)); @@ -911,7 +859,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) emit(__sub(rd, __zero, rd)); return; case OP_lshift: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) { + if (MOP_PAIR_DEST_SRC0(mop)) { emit(__addi(__t0, __zero, 32)); emit(__bltu(rs2, __t0, 20)); emit(__sub(__t1, rs2, __t0)); @@ -936,8 +884,8 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) emit(__sll(rd, rs1, rs2)); return; case OP_rshift: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) { - bool is_unsigned = ph2_ir->src0_is_unsigned; + if (MOP_PAIR_DEST_SRC0(mop)) { + bool is_unsigned = mop->src0_is_unsigned; emit(__addi(__t0, __zero, 32)); emit(__bltu(rs2, __t0, 20)); @@ -965,11 +913,10 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) : __sra(rd_hi, rs1_hi, rs2)); return; } - emit(ph2_ir->src0_is_unsigned ? __srl(rd, rs1, rs2) - : __sra(rd, rs1, rs2)); + emit(mop->src0_is_unsigned ? __srl(rd, rs1, rs2) : __sra(rd, rs1, rs2)); return; case OP_eq: - if (ph2_ir->src0_hi >= 0 && ph2_ir->src1_hi >= 0) { + if (mop->src0_hi >= 0 && mop->src1_hi >= 0) { emit(__xor(__t0, rs1, rs2)); emit(__xor(rd, rs1_hi, rs2_hi)); emit(__or(rd, rd, __t0)); @@ -982,7 +929,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) emit(__xori(rd, rd, 1)); return; case OP_neq: - if (ph2_ir->src0_hi >= 0 && ph2_ir->src1_hi >= 0) { + if (mop->src0_hi >= 0 && mop->src1_hi >= 0) { emit(__xor(__t0, rs1, rs2)); emit(__xor(rd, rs1_hi, rs2_hi)); emit(__or(rd, rd, __t0)); @@ -996,11 +943,10 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) case OP_lt: case OP_geq: case OP_leq: - if (ph2_ir->src0_hi >= 0 && ph2_ir->src1_hi >= 0) { - bool unsigned_cmp = - ph2_ir->src0_is_unsigned || ph2_ir->src1_is_unsigned; - bool reverse = ph2_ir->op == OP_gt || ph2_ir->op == OP_leq; - bool invert = ph2_ir->op == OP_geq || ph2_ir->op == OP_leq; + if (mop->src0_hi >= 0 && mop->src1_hi >= 0) { + bool unsigned_cmp = mop->src0_is_unsigned || mop->src1_is_unsigned; + bool reverse = mop->op == OP_gt || mop->op == OP_leq; + bool invert = mop->op == OP_geq || mop->op == OP_leq; emit(unsigned_cmp ? __sltu(__t0, reverse ? rs2_hi : rs1_hi, reverse ? rs1_hi : rs2_hi) @@ -1016,20 +962,20 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) emit(__xori(rd, rd, 1)); return; } - if (ph2_ir->op == OP_gt || ph2_ir->op == OP_leq) - emit((ph2_ir->src0_is_unsigned || ph2_ir->src1_is_unsigned) + if (mop->op == OP_gt || mop->op == OP_leq) + emit((mop->src0_is_unsigned || mop->src1_is_unsigned) ? __sltu(rd, rs2, rs1) : __slt(rd, rs2, rs1)); else - emit((ph2_ir->src0_is_unsigned || ph2_ir->src1_is_unsigned) + emit((mop->src0_is_unsigned || mop->src1_is_unsigned) ? __sltu(rd, rs1, rs2) : __slt(rd, rs1, rs2)); - if (ph2_ir->op == OP_geq || ph2_ir->op == OP_leq) + if (mop->op == OP_geq || mop->op == OP_leq) emit(__xori(rd, rd, 1)); return; case OP_negate: emit(__sub(rd, __zero, rs1)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) { + if (MOP_PAIR_DEST_SRC0(mop)) { emit(__sltu(__t0, __zero, rs1)); emit(__sub(rd_hi, __zero, rs1_hi)); emit(__sub(rd_hi, rd_hi, __t0)); @@ -1037,29 +983,18 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) return; case OP_bit_not: emit(__xori(rd, rs1, -1)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) + if (MOP_PAIR_DEST_SRC0(mop)) emit(__xori(rd_hi, rs1_hi, -1)); return; case OP_bit_and: - emit(__and(rd, rs1, rs2)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - emit(__and(rd_hi, rs1_hi, rs2_hi)); - return; case OP_bit_or: - emit(__or(rd, rs1, rs2)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - emit(__or(rd_hi, rs1_hi, rs2_hi)); - return; case OP_bit_xor: - emit(__xor(rd, rs1, rs2)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - emit(__xor(rd_hi, rs1_hi, rs2_hi)); + emit(riscv_bitwise_insn(mop->op, rd, rs1, rs2)); + if (MOP_PAIR_BINARY(mop)) + emit(riscv_bitwise_insn(mop->op, rd_hi, rs1_hi, rs2_hi)); return; case OP_log_not: - if (ph2_ir->src0_hi >= 0) { + if (mop->src0_hi >= 0) { emit(__or(__t0, rs1, rs1_hi)); rs1 = __t0; } @@ -1067,18 +1002,18 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) emit(__xori(rd, rd, 1)); return; case OP_trunc: - if (ph2_ir->is_unsigned && (ph2_ir->src1 == 1 || ph2_ir->src1 == 2)) { - int shift = ph2_ir->src1 == 1 ? 24 : 16; + if (mop->is_unsigned && (mop->src1 == 1 || mop->src1 == 2)) { + int shift = mop->src1 == 1 ? 24 : 16; emit(__slli(rd, rs1, shift)); emit(__srli(rd, rd, shift)); - } else if (ph2_ir->src1 == 1) { + } else if (mop->src1 == 1) { emit(__slli(rd, rs1, 24)); emit(__srai(rd, rd, 24)); - } else if (ph2_ir->src1 == 2) { + } else if (mop->src1 == 2) { emit(__slli(rd, rs1, 16)); emit(__srai(rd, rd, 16)); - } else if (ph2_ir->src1 == 4) { + } else if (mop->src1 == 4) { /* No truncation needed for 32-bit values */ emit(__add(rd, rs1, __zero)); } else { @@ -1087,11 +1022,11 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) return; case OP_sign_ext: { /* The upper 16 bits of src1 hold the source size. */ - int source_size = (ph2_ir->src1 >> 16) & 0xFFFF; + int source_size = (mop->src1 >> 16) & 0xFFFF; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi < 0) { + if (mop->dest_hi >= 0 && mop->src0_hi < 0) { if (source_size == 1) { - if (ph2_ir->src0_is_unsigned || ph2_ir->src0_is_pointer) + if (mop->src0_is_unsigned || mop->src0_is_pointer) emit(__andi(rd, rs1, 0xFF)); else { emit(__slli(rd, rs1, 24)); @@ -1099,12 +1034,12 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) } } else if (source_size == 2) { emit(__slli(rd, rs1, 16)); - emit(ph2_ir->src0_is_unsigned || ph2_ir->src0_is_pointer + emit(mop->src0_is_unsigned || mop->src0_is_pointer ? __srli(rd, rd, 16) : __srai(rd, rd, 16)); } else emit(__addi(rd, rs1, 0)); - if (ph2_ir->src0_is_unsigned || ph2_ir->src0_is_pointer) + if (mop->src0_is_unsigned || mop->src0_is_pointer) emit(__addi(rd_hi, __zero, 0)); else emit(__srai(rd_hi, rd, 31)); @@ -1124,14 +1059,14 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) * for RISC-V immediate field */ emit(__slli(rd, rs1, shift_amount)); - emit(ph2_ir->src0_is_unsigned ? __srli(rd, rd, shift_amount) - : __srai(rd, rd, shift_amount)); + emit(mop->src0_is_unsigned ? __srli(rd, rd, shift_amount) + : __srai(rd, rd, shift_amount)); } else { /* Fallback for other sizes */ emit(__andi(rd, rs1, 0xFF)); emit(__slli(rd, rd, shift_amount)); - emit(ph2_ir->src0_is_unsigned ? __srli(rd, rd, shift_amount) - : __srai(rd, rd, shift_amount)); + emit(mop->src0_is_unsigned ? __srli(rd, rd, shift_amount) + : __srai(rd, rd, shift_amount)); } return; } @@ -1140,11 +1075,11 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) * when casting between two direct wide scalar types. */ emit(__addi(rd, rs1, 0)); - if (ph2_ir->dest_hi >= 0) { - if (ph2_ir->src0_hi >= 0) { - if (ph2_ir->dest_hi != ph2_ir->src0_hi) + if (mop->dest_hi >= 0) { + if (mop->src0_hi >= 0) { + if (mop->dest_hi != mop->src0_hi) emit(__addi(rd_hi, rs1_hi, 0)); - } else if (ph2_ir->src0_is_unsigned || ph2_ir->src0_is_pointer) + } else if (mop->src0_is_unsigned || mop->src0_is_pointer) emit(__addi(rd_hi, __zero, 0)); else emit(__srai(rd_hi, rs1, 31)); @@ -1158,6 +1093,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) void plt_generate(void); void code_generate(void) { + int global_stack_size; int ofs; if (dynlink) { @@ -1192,8 +1128,7 @@ void code_generate(void) * __libc_start_main(main_wrapper, argc, argv, NULL, * NULL, NULL, stack_end) */ - emit(__lui(__a0, rv_hi(elf_code_start + 40))); - emit(__addi(__a0, __a0, rv_lo(elf_code_start + 40))); + emit_rv32_split_address(false, __a0, __a0, elf_code_start + 40); emit(__lw(__a1, __sp, 0)); emit(__addi(__a2, __sp, 4)); emit(__addi(__a3, __zero, 0)); @@ -1206,8 +1141,7 @@ void code_generate(void) */ ofs = (dynamic_sections.elf_plt_start + PLT_FIXUP_SIZE) - (elf_code_start + elf_code->size); - emit(__auipc(__ra, rv_hi(ofs))); - emit(__jalr(__ra, __ra, rv_lo(ofs))); + emit_rv32_split_address(true, __ra, __ra, ofs); /* The main wrapper is located here under the dynamic linking mode * @@ -1224,14 +1158,15 @@ void code_generate(void) emit(__sw(__s0, __sp, 0)); /* callee-saved */ emit(__addi(__s0, __a0, 0)); /* argc */ emit(__addi(__s1, __a1, 0)); /* argv */ - ofs = ALIGN_UP(GLOBAL_FUNC->stack_size, RV32_ALIGNMENT) + 4; + global_stack_size = + ALIGN_UP(GLOBAL_FUNC->stack_size, RV32_ALIGNMENT) + 4; } else { /* When using static linking, the starting address of the main wrapper * is here. * * Save original sp in s0 first. */ - ofs = ALIGN_UP(GLOBAL_FUNC->stack_size, RV32_ALIGNMENT); + global_stack_size = ALIGN_UP(GLOBAL_FUNC->stack_size, RV32_ALIGNMENT); emit(__addi(__s0, __sp, 0)); } @@ -1240,8 +1175,7 @@ void code_generate(void) * 2. jump to global init function * 3. call the main function */ - emit(__lui(__t0, rv_hi(ofs))); - emit(__addi(__t0, __t0, rv_lo(ofs))); + emit_rv32_split_address(false, __t0, __t0, global_stack_size); emit(__sub(__sp, __sp, __t0)); emit(__addi(__gp, __sp, 0)); /* Set up global pointer */ @@ -1258,7 +1192,8 @@ void code_generate(void) emit(__sw(__zero, __t1, 0)); emit(__bne(__t0, __zero, -12)); } - emit(__jal(__ra, GLOBAL_FUNC->bbs->elf_offset - elf_code->size)); + ofs = GLOBAL_FUNC->bbs->elf_offset - elf_code->size; + emit_rv32_split_address(true, __ra, __ra, ofs); if (!dynlink) { /* syscall trampoline for __syscall */ @@ -1273,24 +1208,21 @@ void code_generate(void) emit(__jalr(__zero, __ra, 0)); } - ph2_ir_t *ph2_ir; - for (ph2_ir = GLOBAL_FUNC->bbs->ph2_ir_list.head; ph2_ir; - ph2_ir = ph2_ir->next) - emit_ph2_ir(ph2_ir); + mop_visit_globals(emit_mop); /* prepare 'argc' and 'argv', then proceed to 'main' function */ if (MAIN_BB) { if (dynlink) { emit(__addi(__a0, __s0, 0)); emit(__addi(__a1, __s1, 0)); - emit(__jal(__ra, MAIN_BB->elf_offset - elf_code->size)); + ofs = MAIN_BB->elf_offset - elf_code->size; + emit_rv32_split_address(true, __ra, __ra, ofs); /* - Restore sp, s0 and s1. * - Transfer control back to __libc_start_main() using * the preserved ra. */ - emit(__lui(__t0, rv_hi(ofs))); - emit(__addi(__t0, __t0, rv_lo(ofs))); + emit_rv32_split_address(false, __t0, __t0, global_stack_size); emit(__add(__sp, __sp, __t0)); emit(__lw(__ra, __sp, 8)); emit(__lw(__s1, __sp, 4)); @@ -1302,7 +1234,8 @@ void code_generate(void) emit(__addi(__t0, __s0, 0)); emit(__lw(__a0, __t0, 0)); emit(__addi(__a1, __t0, 4)); - emit(__jal(__ra, MAIN_BB->elf_offset - elf_code->size)); + ofs = MAIN_BB->elf_offset - elf_code->size; + emit_rv32_split_address(true, __ra, __ra, ofs); /* exit with main's return value in a0 */ emit(__addi(__a7, __zero, 93)); @@ -1310,9 +1243,9 @@ void code_generate(void) } } - for (int i = 0; i < ph2_ir_idx; i++) { - ph2_ir = PH2_IR_FLATTEN[i]; - emit_ph2_ir(ph2_ir); + for (int i = 0; i < mop_flat_count; i++) { + vir_mop_t *mop = MOP_FLAT[i]; + emit_mop(mop); } } diff --git a/src/riscv.c b/src/riscv.c index 82cd9d1c..1ff146dc 100644 --- a/src/riscv.c +++ b/src/riscv.c @@ -95,7 +95,8 @@ typedef enum { int rv_extract_bits(int imm, int i_start, int i_end, int d_start, int d_end) { - int v; + /* Unsigned, so a field placed in bit 31 does not overflow a signed int. */ + unsigned int v; if (d_end - d_start != i_end - i_start || i_start > i_end || d_start > d_end) @@ -122,27 +123,22 @@ int rv_encode_R(rv_op op, rv_reg rd, rv_reg rs1, rv_reg rs2) return op + (rd << 7) + (rs1 << 15) + (rs2 << 20); } -int rv_encode_I(rv_op op, rv_reg rd, rv_reg rs1, int imm) +static int rv_imm12(int imm) { if (imm > 2047 || imm < -2048) fatal("Offset too large"); + return imm < 0 ? imm + 4096 : imm; +} - if (imm < 0) { - imm += 4096; - imm &= (1 << 13) - 1; - } - return op + (rd << 7) + (rs1 << 15) + (imm << 20); +int rv_encode_I(rv_op op, rv_reg rd, rv_reg rs1, int imm) +{ + imm = rv_imm12(imm); + return op + (rd << 7) + (rs1 << 15) + ((unsigned int) imm << 20); } int rv_encode_S(rv_op op, rv_reg rs1, rv_reg rs2, int imm) { - if (imm > 2047 || imm < -2048) - fatal("Offset too large"); - - if (imm < 0) { - imm += 4096; - imm &= (1 << 13) - 1; - } + imm = rv_imm12(imm); return op + (rs1 << 15) + (rs2 << 20) + rv_extract_bits(imm, 0, 4, 7, 11) + rv_extract_bits(imm, 5, 11, 25, 31); } @@ -160,7 +156,7 @@ int rv_encode_B(rv_op op, rv_reg rs1, rv_reg rs2, int imm) return op + (rs1 << 15) + (rs2 << 20) + rv_extract_bits(imm, 11, 11, 7, 7) + rv_extract_bits(imm, 1, 4, 8, 11) + - rv_extract_bits(imm, 5, 10, 25, 30) + (sign << 31); + rv_extract_bits(imm, 5, 10, 25, 30) + ((unsigned int) sign << 31); } int rv_encode_J(rv_op op, rv_reg rd, int imm) @@ -178,7 +174,7 @@ int rv_encode_J(rv_op op, rv_reg rd, int imm) } return op + (rd << 7) + rv_extract_bits(imm, 1, 10, 21, 30) + rv_extract_bits(imm, 11, 11, 20, 20) + - rv_extract_bits(imm, 12, 19, 12, 19) + (sign << 31); + rv_extract_bits(imm, 12, 19, 12, 19) + ((unsigned int) sign << 31); } int rv_encode_U(rv_op op, rv_reg rd, int imm) @@ -186,193 +182,82 @@ int rv_encode_U(rv_op op, rv_reg rd, int imm) return op + (rd << 7) + rv_extract_bits(imm, 12, 31, 12, 31); } -int __add(rv_reg rd, rv_reg rs1, rv_reg rs2) -{ - return rv_encode_R(rv_add, rd, rs1, rs2); -} - -int __sub(rv_reg rd, rv_reg rs1, rv_reg rs2) -{ - return rv_encode_R(rv_sub, rd, rs1, rs2); -} - -int __mulhu(rv_reg rd, rv_reg rs1, rv_reg rs2) -{ - return rv_encode_R(rv_mulhu, rd, rs1, rs2); -} - -int __xor(rv_reg rd, rv_reg rs1, rv_reg rs2) -{ - return rv_encode_R(rv_xor, rd, rs1, rs2); -} - -int __or(rv_reg rd, rv_reg rs1, rv_reg rs2) -{ - return rv_encode_R(rv_or, rd, rs1, rs2); -} - -int __and(rv_reg rd, rv_reg rs1, rv_reg rs2) -{ - return rv_encode_R(rv_and, rd, rs1, rs2); -} - -int __sll(rv_reg rd, rv_reg rs1, rv_reg rs2) -{ - return rv_encode_R(rv_sll, rd, rs1, rs2); -} - -int __srl(rv_reg rd, rv_reg rs1, rv_reg rs2) -{ - return rv_encode_R(rv_srl, rd, rs1, rs2); -} - -int __sra(rv_reg rd, rv_reg rs1, rv_reg rs2) -{ - return rv_encode_R(rv_sra, rd, rs1, rs2); -} - -int __slt(rv_reg rd, rv_reg rs1, rv_reg rs2) -{ - return rv_encode_R(rv_slt, rd, rs1, rs2); -} - -int __sltu(rv_reg rd, rv_reg rs1, rv_reg rs2) -{ - return rv_encode_R(rv_sltu, rd, rs1, rs2); -} - -int __addi(rv_reg rd, rv_reg rs1, int imm) -{ - return rv_encode_I(rv_addi, rd, rs1, imm); -} - -int __xori(rv_reg rd, rv_reg rs1, int imm) -{ - return rv_encode_I(rv_xori, rd, rs1, imm); -} - -int __andi(rv_reg rd, rv_reg rs1, int imm) -{ - return rv_encode_I(rv_andi, rd, rs1, imm); -} - -int __slli(rv_reg rd, rv_reg rs1, int imm) -{ - return rv_encode_I(rv_slli, rd, rs1, imm); -} - -int __srli(rv_reg rd, rv_reg rs1, int imm) -{ - return rv_encode_I(rv_srli, rd, rs1, imm); -} - -int __srai(rv_reg rd, rv_reg rs1, int imm) -{ - return rv_encode_I(rv_srai, rd, rs1, imm); -} - -int __lb(rv_reg rd, rv_reg rs1, int imm) -{ - return rv_encode_I(rv_lb, rd, rs1, imm); -} - -int __lh(rv_reg rd, rv_reg rs1, int imm) -{ - return rv_encode_I(rv_lh, rd, rs1, imm); -} - -int __lbu(rv_reg rd, rv_reg rs1, int imm) -{ - return rv_encode_I(rv_lbu, rd, rs1, imm); -} - -int __lhu(rv_reg rd, rv_reg rs1, int imm) -{ - return rv_encode_I(rv_lhu, rd, rs1, imm); -} - -int __lw(rv_reg rd, rv_reg rs1, int imm) -{ - return rv_encode_I(rv_lw, rd, rs1, imm); -} - -int __sb(rv_reg rd, rv_reg rs1, int imm) -{ - return rv_encode_S(rv_sb, rs1, rd, imm); -} - -int __sh(rv_reg rd, rv_reg rs1, int imm) -{ - return rv_encode_S(rv_sh, rs1, rd, imm); -} - -int __sw(rv_reg rd, rv_reg rs1, int imm) -{ - return rv_encode_S(rv_sw, rs1, rd, imm); -} - -int __beq(rv_reg rs1, rv_reg rs2, int imm) -{ - return rv_encode_B(rv_beq, rs1, rs2, imm); -} - -int __bne(rv_reg rs1, rv_reg rs2, int imm) -{ - return rv_encode_B(rv_bne, rs1, rs2, imm); -} - -int __bltu(rv_reg rs1, rv_reg rs2, int imm) -{ - return rv_encode_B(rv_bltu, rs1, rs2, imm); -} +/* Typed wrappers share the encoding-format argument order. */ +#define RV_INSN_R(name, op) \ + int name(rv_reg rd, rv_reg rs1, rv_reg rs2) \ + { \ + return rv_encode_R(op, rd, rs1, rs2); \ + } +#define RV_INSN_I(name, op) \ + int name(rv_reg rd, rv_reg rs1, int imm) \ + { \ + return rv_encode_I(op, rd, rs1, imm); \ + } +#define RV_INSN_S(name, op) \ + int name(rv_reg rd, rv_reg rs1, int imm) \ + { \ + return rv_encode_S(op, rs1, rd, imm); \ + } +#define RV_INSN_B(name, op) \ + int name(rv_reg rs1, rv_reg rs2, int imm) \ + { \ + return rv_encode_B(op, rs1, rs2, imm); \ + } +#define RV_INSN_U(name, op) \ + int name(rv_reg rd, int imm) \ + { \ + return rv_encode_U(op, rd, imm); \ + } +RV_INSN_R(__add, rv_add) +RV_INSN_R(__sub, rv_sub) +RV_INSN_R(__mulhu, rv_mulhu) +RV_INSN_R(__xor, rv_xor) +RV_INSN_R(__or, rv_or) +RV_INSN_R(__and, rv_and) +RV_INSN_R(__sll, rv_sll) +RV_INSN_R(__srl, rv_srl) +RV_INSN_R(__sra, rv_sra) +RV_INSN_R(__slt, rv_slt) +RV_INSN_R(__sltu, rv_sltu) +RV_INSN_I(__addi, rv_addi) +RV_INSN_I(__xori, rv_xori) +RV_INSN_I(__andi, rv_andi) +RV_INSN_I(__slli, rv_slli) +RV_INSN_I(__srli, rv_srli) +RV_INSN_I(__srai, rv_srai) +RV_INSN_I(__lb, rv_lb) +RV_INSN_I(__lh, rv_lh) +RV_INSN_I(__lbu, rv_lbu) +RV_INSN_I(__lhu, rv_lhu) +RV_INSN_I(__lw, rv_lw) +RV_INSN_S(__sb, rv_sb) +RV_INSN_S(__sh, rv_sh) +RV_INSN_S(__sw, rv_sw) +RV_INSN_B(__beq, rv_beq) +RV_INSN_B(__bne, rv_bne) +RV_INSN_B(__bltu, rv_bltu) int __jal(rv_reg rd, int imm) { return rv_encode_J(rv_jal, rd, imm); } - -int __jalr(rv_reg rd, rv_reg rs1, int imm) -{ - return rv_encode_I(rv_jalr, rd, rs1, imm); -} - -int __lui(rv_reg rd, int imm) -{ - return rv_encode_U(rv_lui, rd, imm); -} - -int __auipc(rv_reg rd, int imm) -{ - return rv_encode_U(rv_auipc, rd, imm); -} +RV_INSN_I(__jalr, rv_jalr) +RV_INSN_U(__lui, rv_lui) +RV_INSN_U(__auipc, rv_auipc) int __ecall(void) { return rv_encode_I(rv_ecall, __zero, __zero, 0); } +RV_INSN_R(__mul, rv_mul) +RV_INSN_R(__div, rv_div) +RV_INSN_R(__divu, rv_divu) +RV_INSN_R(__mod, rv_mod) +RV_INSN_R(__modu, rv_modu) -int __mul(rv_reg rd, rv_reg rs1, rv_reg rs2) -{ - return rv_encode_R(rv_mul, rd, rs1, rs2); -} - -int __div(rv_reg rd, rv_reg rs1, rv_reg rs2) -{ - return rv_encode_R(rv_div, rd, rs1, rs2); -} - -int __divu(rv_reg rd, rv_reg rs1, rv_reg rs2) -{ - return rv_encode_R(rv_divu, rd, rs1, rs2); -} - -int __mod(rv_reg rd, rv_reg rs1, rv_reg rs2) -{ - return rv_encode_R(rv_mod, rd, rs1, rs2); -} - -int __modu(rv_reg rd, rv_reg rs1, rv_reg rs2) -{ - return rv_encode_R(rv_modu, rd, rs1, rs2); -} +#undef RV_INSN_R +#undef RV_INSN_I +#undef RV_INSN_S +#undef RV_INSN_B +#undef RV_INSN_U diff --git a/src/ssa.c b/src/ssa.c deleted file mode 100644 index fc32239c..00000000 --- a/src/ssa.c +++ /dev/null @@ -1,4913 +0,0 @@ -/* - * shecc - Self-Hosting and Educational C Compiler. - * - * shecc is freely redistributable under the BSD 2 clause license. See the file - * "LICENSE" for information on usage and redistribution of this file. - */ -#include -#include - -#include "defs.h" -#include "globals.c" - -/* Constant cast optimization. Despite the file name this is not SCCP: there is - * no lattice and no CFG-edge worklist anywhere in the tree. - */ -#include "opt-sccp.c" - -/* Configuration constants - replace magic numbers */ -#define PHI_WORKLIST_SIZE 128 -#define DCE_WORKLIST_SIZE 2048 - -/* Dead store elimination window size */ -#define OVERWRITE_WINDOW 3 - -/* Whether @var lives in memory that other code can reach without naming it: a - * global, or a variable whose address is taken. SSA gives such a variable one - * name, and a write to it is observable however unused it looks. - */ -bool var_in_memory(const var_t *var) -{ - return var->is_global || var->address_taken; -} - -/* Whether @insn accesses a volatile object, which C99 6.7.3p6 counts as a side - * effect whether or not anything uses the value: it names such an object as an - * operand, or reads through an address mark_volatile_addresses() traced to one. - * Taking an object's address does not access it. - */ -bool insn_reads_volatile(const insn_t *insn) -{ - if (insn->opcode == OP_address_of || insn->opcode == OP_allocat) - return false; - if (insn->opcode == OP_read && insn->rs1 && insn->rs1->is_volatile) - return !insn->rd || !insn->rd->is_assignment_reload; - return var_is_volatile_object(insn->rs1) || - var_is_volatile_object(insn->rs2) || - var_is_volatile_object(insn->rs3); -} - -/* Whether @var holds an address into a volatile object: a volatile pointer or - * array, or a temporary already traced to one. - */ -bool var_is_volatile_address(const var_t *var) -{ - if (!var || !var->is_volatile) - return false; - return var->ptr_level || var->array_size || - (var->type && var->type->ptr_level) || var->var_name[0] == '.'; -} - -/* Mark every temporary that holds an address into a volatile object. - * - * A declaration says its object is volatile, but a member or an element of the - * object, or whatever a pointer to volatile designates, is read through an - * address the parser computed into a temporary, and nothing on the temporary - * says what it points into. Follow each address from the object or pointer it - * was derived from, so that the read at the end of the chain is recognizable as - * the side effect it is. Only addresses are followed: an int computed from a - * volatile int is an ordinary value. - */ -void mark_volatile_addresses(func_t *func) -{ - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - var_t *rd = insn->rd; - bool from_volatile = false; - - if (!rd || rd->is_volatile || rd->var_name[0] != '.') - continue; - - switch (insn->opcode) { - case OP_address_of: - from_volatile = insn->rs1 && insn->rs1->is_volatile; - break; - case OP_add: - case OP_sub: - from_volatile = var_is_volatile_address(insn->rs1) || - var_is_volatile_address(insn->rs2); - break; - case OP_assign: - case OP_cast: - from_volatile = var_is_volatile_address(insn->rs1); - break; - default: - break; - } - if (from_volatile) - rd->is_volatile = true; - } - } -} - -void var_list_ensure_capacity(var_list_t *list, int min_capacity) -{ - if (list->capacity >= min_capacity) - return; - - int new_capacity = list->capacity ? list->capacity : HOST_PTR_SIZE; - - while (new_capacity < min_capacity) - new_capacity <<= 1; - - /* arena_realloc() extends the block in place when this list was the last - * thing allocated, which the liveness sets often are; allocating a fresh - * array and copying abandoned the old one in the arena every time. - */ - list->elements = arena_realloc(BB_ARENA, (char *) list->elements, - list->capacity * HOST_PTR_SIZE, - new_capacity * HOST_PTR_SIZE); - list->capacity = new_capacity; -} - -/* Whether @var appears in @list. */ -bool var_list_holds(const var_list_t *list, var_t *var) -{ - for (int i = 0; i < list->size; i++) { - if (list->elements[i] == var) - return true; - } - return false; -} - -void var_list_add_var(var_list_t *list, var_t *var) -{ - if (var_list_holds(list, var)) - return; - - var_list_ensure_capacity(list, list->size + 1); - list->elements[list->size++] = var; -} - -/* Append without the membership scan var_list_add_var() performs. Callers that - * use this must establish uniqueness themselves. - */ -void var_list_append(var_list_t *list, var_t *var) -{ - var_list_ensure_capacity(list, list->size + 1); - list->elements[list->size++] = var; -} - -void var_list_assign_array(var_list_t *list, var_t **data, int count) -{ - var_list_ensure_capacity(list, count); - memcpy(list->elements, data, count * HOST_PTR_SIZE); - list->size = count; -} - -/* cfront does not accept structure as an argument, pass pointer The only thing - * a step of either traversal changes in the argument block is the block it - * names, so a step sets that field and puts it back on the way out. Copying the - * whole structure per edge instead -- which is what these did -- costs a copy - * on every block of every traversal, and the traversals are how nearly every - * analysis in the middle end walks a function. - */ -void bb_forward_traversal(bb_traversal_args_t *args) -{ - basic_block_t *bb = args->bb; - func_t *func = args->func; - - bb->visited = func->visited; - - if (args->preorder_cb) - args->preorder_cb(func, bb); - - if (bb->next) { - if (bb->next->visited < func->visited) { - args->bb = bb->next; - bb_forward_traversal(args); - } - } - if (bb->then_) { - if (bb->then_->visited < func->visited) { - args->bb = bb->then_; - bb_forward_traversal(args); - } - } - if (bb->else_) { - if (bb->else_->visited < func->visited) { - args->bb = bb->else_; - bb_forward_traversal(args); - } - } - - args->bb = bb; - - if (args->postorder_cb) - args->postorder_cb(func, bb); -} - -/* cfront does not accept structure as an argument, pass pointer */ -void bb_backward_traversal(bb_traversal_args_t *args) -{ - basic_block_t *bb = args->bb; - func_t *func = args->func; - - bb->visited = func->visited; - - if (args->preorder_cb) - args->preorder_cb(func, bb); - - for (int i = 0; i < bb->prev_idx; i++) { - basic_block_t *pred = bb->prev[i].bb; - - if (!pred) - continue; - if (pred->visited < func->visited) { - args->bb = pred; - bb_backward_traversal(args); - } - } - - args->bb = bb; - - if (args->postorder_cb) - args->postorder_cb(func, bb); -} - -void bb_index_rpo(func_t *func, basic_block_t *bb) -{ - bb->rpo = func->bb_cnt++; -} - -void bb_reverse_index(func_t *func, basic_block_t *bb) -{ - bb->rpo = func->bb_cnt - bb->rpo; -} - -void bb_build_rpo(func_t *func, basic_block_t *bb) -{ - if (func->bbs == bb) - return; - - basic_block_t *prev = func->bbs; - basic_block_t *curr = prev->rpo_next; - for (; curr; curr = curr->rpo_next) { - if (curr->rpo < bb->rpo) { - prev = curr; - continue; - } - bb->rpo_next = curr; - prev->rpo_next = bb; - return; - } - - prev->rpo_next = bb; -} - -void build_rpo(void) -{ - bb_traversal_args_t *args = arena_alloc_traversal_args(); - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - args->func = func; - args->bb = func->bbs; - - func->visited++; - args->postorder_cb = bb_index_rpo; - bb_forward_traversal(args); - - func->visited++; - args->postorder_cb = bb_reverse_index; - bb_forward_traversal(args); - - func->visited++; - args->postorder_cb = bb_build_rpo; - bb_forward_traversal(args); - } -} - -basic_block_t *intersect(basic_block_t *i, basic_block_t *j) -{ - while (i != j) { - while (i->rpo > j->rpo) - i = i->idom; - while (j->rpo > i->rpo) - j = j->idom; - } - return i; -} - -/* Find the immediate dominator of each basic block to build the dominator tree. - * - * Once the dominator tree is built, we can perform the more advanced - * optimiaztion according to the liveness analysis and the reachability - * analysis, e.g. common subexpression elimination, loop optimiaztion or dead - * code elimination . - * - * Reference: - * Cooper, Keith D.; Harvey, Timothy J.; Kennedy, Ken (2001). - * "A Simple, Fast Dominance Algorithm" - */ -void build_idom(void) -{ - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - bool changed; - - func->bbs->idom = func->bbs; - - do { - changed = false; - - for (basic_block_t *bb = func->bbs->rpo_next; bb; - bb = bb->rpo_next) { - /* pick one predecessor */ - basic_block_t *pred = NULL; - for (int i = 0; i < bb->prev_idx; i++) { - if (!bb->prev[i].bb) - continue; - if (!bb->prev[i].bb->idom) - continue; - pred = bb->prev[i].bb; - break; - } - - /* Reverse postorder puts a predecessor of every reachable block - * ahead of it, so one is normally settled by now. A block where - * none is cannot be given an immediate dominator yet; leaving - * it for a later round is what keeps the walk off an - * uninitialised pointer. - */ - if (!pred) - continue; - - for (int i = 0; i < bb->prev_idx; i++) { - if (!bb->prev[i].bb) - continue; - if (bb->prev[i].bb == pred) - continue; - if (bb->prev[i].bb->idom) - pred = intersect(bb->prev[i].bb, pred); - } - if (bb->idom != pred) { - bb->idom = pred; - changed = true; - } - } - } while (changed); - } -} - -bool dom_connect(basic_block_t *pred, basic_block_t *succ) -{ - if (succ->dom_prev) - return false; - - for (int i = 0; i < pred->dom_next_idx; i++) { - if (pred->dom_next[i] == succ) - return false; - } - - if (pred->dom_next_idx >= pred->dom_next_cap) - pred->dom_next = - arena_grow(BB_ARENA, (char *) pred->dom_next, &pred->dom_next_cap, - sizeof(basic_block_t *), 4, MAX_BB_DOM_SUCC, - "Too many children in dominator tree"); - - pred->dom_next[pred->dom_next_idx++] = succ; - succ->dom_prev = pred; - return true; -} - -void bb_build_dom(func_t *func, basic_block_t *bb) -{ - basic_block_t *curr = bb; - while (curr != func->bbs) { - if (!dom_connect(curr->idom, curr)) - break; - curr = curr->idom; - } -} - -void build_dom(void) -{ - bb_traversal_args_t *args = arena_alloc_traversal_args(); - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - args->func = func; - args->bb = func->bbs; - - func->visited++; - args->preorder_cb = bb_build_dom; - bb_forward_traversal(args); - } -} - -/* Recompute the dominator tree over the CFG as it now stands. - * - * if_convert() only removes edges, and losing one can only strengthen - * dominance, so a stale tree stays conservative there. thread_const_branch() - * adds "pred -> target", a path that bypasses the join: a join that dominated - * the target no longer does. strength_reduce() and mark_loop_depth() both ask - * is_dominate() which edges close a loop, and against a stale tree a forward - * edge reads as a back edge -- mark_natural_loop() then walks predecessors out - * of the region it was meant to stay inside, since it has no stop at the header - * and relies on the header dominating the latch. - * - * The order has to be rebuilt along with the tree. Both passes keep the - * rpo_next chain consistent -- a block they drop comes out of it -- but - * dropping blocks and adding edges does not preserve reverse postorder, and - * build_idom() depends on it: it reaches each block once a predecessor already - * has an idom, and out of order it can meet one where none does. - */ -void rebuild_dom(void) -{ - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - if (!func->bbs) - continue; - - /* dom_connect() refuses a block that already has a parent, and - * bb_build_rpo() splices into the chain rather than replacing it, so - * both have to be taken down before they can be built again. The next - * link is read before it is cleared: clearing as the walk goes would - * cut the chain out from under it. - */ - basic_block_t *bb = func->bbs; - - while (bb) { - basic_block_t *next = bb->rpo_next; - - bb->idom = NULL; - bb->dom_prev = NULL; - bb->dom_next_idx = 0; - bb->rpo_next = NULL; - bb = next; - } - func->bb_cnt = 0; - } - - build_rpo(); - build_idom(); - build_dom(); -} - -void bb_build_df(func_t *func, basic_block_t *bb) -{ - UNUSED(func); - - int cnt = 0; - for (int i = 0; i < bb->prev_idx; i++) { - if (bb->prev[i].bb) - cnt++; - } - if (cnt <= 0) - return; - - for (int i = 0; i < bb->prev_idx; i++) { - if (bb->prev[i].bb) { - /* Walk up from the predecessor to this block's immediate dominator. - * The walk normally stops there, since a block's immediate - * dominator dominates all of its predecessors -- but an edge the - * dominator tree does not account for runs off the top instead, so - * stop at the root as well. Ending early only widens the frontier, - * which costs a phi that turns out to be trivial. - */ - for (basic_block_t *curr = bb->prev[i].bb; curr && curr != bb->idom; - curr = curr->idom) - bb_add_df(curr, bb); - } - } -} - -void build_df(void) -{ - bb_traversal_args_t *args = arena_alloc_traversal_args(); - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - args->func = func; - args->bb = func->bbs; - - func->visited++; - args->postorder_cb = bb_build_df; - bb_forward_traversal(args); - } -} - -basic_block_t *reverse_intersect(basic_block_t *i, basic_block_t *j) -{ - while (i != j) { - while (i->rpo_r > j->rpo_r) - i = i->r_idom; - while (j->rpo_r > i->rpo_r) - j = j->r_idom; - } - return i; -} - -void build_r_idom(void) -{ - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - bool changed; - - func->exit->r_idom = func->exit; - - do { - changed = false; - - for (basic_block_t *bb = func->exit->rpo_r_next; bb; - bb = bb->rpo_r_next) { - /* pick one predecessor */ - basic_block_t *pred = NULL; - if (bb->next && bb->next->r_idom) { - pred = bb->next; - } else if (bb->else_ && bb->else_->r_idom) { - pred = bb->else_; - } else if (bb->then_ && bb->then_->r_idom) { - pred = bb->then_; - } - - /* The mirror of the rule in build_idom(): reverse postorder - * from the exit puts a successor of every block that reaches it - * ahead of that block, so one is normally settled by now. A - * block where none is cannot be given an immediate - * postdominator yet; leaving it for a later round is what keeps - * the walk off an uninitialised pointer. - */ - if (!pred) - continue; - - if (bb->next && bb->next != pred && bb->next->r_idom) - pred = reverse_intersect(bb->next, pred); - if (bb->else_ && bb->else_ != pred && bb->else_->r_idom) - pred = reverse_intersect(bb->else_, pred); - if (bb->then_ && bb->then_ != pred && bb->then_->r_idom) - pred = reverse_intersect(bb->then_, pred); - if (bb->r_idom != pred) { - bb->r_idom = pred; - changed = true; - } - } - } while (changed); - } -} - -bool rdom_connect(basic_block_t *pred, basic_block_t *succ) -{ - /* A block is attached to the reverse-dominator tree at most once, so the - * rdom_prev guard above already rules out a duplicate edge. - */ - if (succ->rdom_prev) - return false; - - /* rdom_count only ever guarded a MAX_BB_RDOM_SUCC ceiling on an array of - * reverse-dominator children that no longer exists -- nothing reads the - * count, and nothing walks those children -- so the field is gone. - */ - succ->rdom_prev = pred; - return true; -} - -void bb_build_rdom(func_t *func, basic_block_t *bb) -{ - if (!func->bbs) - return; - for (basic_block_t *curr = bb; curr != func->exit; curr = curr->r_idom) { - if (!rdom_connect(curr->r_idom, curr)) - break; - } -} - -void build_rdom(void) -{ - bb_traversal_args_t *args = arena_alloc_traversal_args(); - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - args->func = func; - args->bb = func->exit; - - func->visited++; - args->preorder_cb = bb_build_rdom; - bb_backward_traversal(args); - } -} - -void bb_build_rdf(func_t *func, basic_block_t *bb) -{ - UNUSED(func); - - int cnt = 0; - if (bb->next) - cnt++; - if (bb->then_) - cnt++; - if (bb->else_) - cnt++; - if (cnt <= 0) - return; - - /* As in bb_build_df(), the walk up the post-dominator tree stops at this - * block's immediate post-dominator, and at the root when an edge the tree - * does not account for takes it past that. - */ - if (bb->next) { - for (basic_block_t *curr = bb->next; curr && curr != bb->r_idom; - curr = curr->r_idom) - bb_add_rdf(curr, bb); - } - if (bb->else_) { - for (basic_block_t *curr = bb->else_; curr && curr != bb->r_idom; - curr = curr->r_idom) - bb_add_rdf(curr, bb); - } - if (bb->then_) { - for (basic_block_t *curr = bb->then_; curr && curr != bb->r_idom; - curr = curr->r_idom) - bb_add_rdf(curr, bb); - } -} - -void build_rdf(void) -{ - bb_traversal_args_t *args = arena_alloc_traversal_args(); - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - args->func = func; - args->bb = func->exit; - - func->visited++; - args->postorder_cb = bb_build_rdf; - bb_backward_traversal(args); - } -} - -void use_chain_add_tail(insn_t *i, var_t *var) -{ - use_chain_t *u = arena_calloc(INSN_ARENA, 1, sizeof(use_chain_t)); - - u->insn = i; - if (!var->users_head) - var->users_head = u; - else - var->users_tail->next = u; - u->prev = var->users_tail; - var->users_tail = u; -} - -void use_chain_build(void) -{ - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (insn_t *i = bb->insn_list.head; i; i = i->next) { - if (i->rs1) - use_chain_add_tail(i, i->rs1); - if (i->rs2) - use_chain_add_tail(i, i->rs2); - } - } - } -} - -bool var_check_killed(const var_t *var, const basic_block_t *bb) -{ - for (int i = 0; i < bb->live_kill.size; i++) { - if (bb->live_kill.elements[i] == var) - return true; - } - return false; -} - -void bb_add_killed_var(basic_block_t *bb, var_t *var) -{ - var_list_add_var(&bb->live_kill, var); -} - -void var_add_killed_bb(var_t *var, basic_block_t *bb) -{ - bool found = false; - ref_block_t *ref; - for (ref = var->ref_block_list.head; ref; ref = ref->next) { - if (ref->bb == bb) { - found = true; - break; - } - } - if (found) - return; - - ref = arena_calloc(GENERAL_ARENA, 1, sizeof(ref_block_t)); - ref->bb = bb; - if (!var->ref_block_list.head) - var->ref_block_list.head = ref; - else - var->ref_block_list.tail->next = ref; - - var->ref_block_list.tail = ref; -} - -void fn_add_global(func_t *func, var_t *var) -{ - bool found = false; - symbol_t *sym; - for (sym = func->global_sym_list.head; sym; sym = sym->next) { - if (sym->var == var) { - found = true; - break; - } - } - if (found) - return; - - sym = arena_alloc_symbol(); - sym->var = var; - if (!func->global_sym_list.head) { - sym->index = 0; - func->global_sym_list.head = sym; - func->global_sym_list.tail = sym; - } else { - sym->index = func->global_sym_list.tail->index + 1; - func->global_sym_list.tail->next = sym; - func->global_sym_list.tail = sym; - } -} - -void bb_solve_globals(func_t *func, basic_block_t *bb) -{ - UNUSED(func); - - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->rs1) - if (!var_check_killed(insn->rs1, bb)) - fn_add_global(bb->belong_to, insn->rs1); - if (insn->rs2) - if (!var_check_killed(insn->rs2, bb)) - fn_add_global(bb->belong_to, insn->rs2); - if (insn->rd) { - bb_add_killed_var(bb, insn->rd); - var_add_killed_bb(insn->rd, bb); - } - } -} - -void solve_globals(void) -{ - bb_traversal_args_t *args = arena_alloc_traversal_args(); - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - args->func = func; - args->bb = func->bbs; - - func->visited++; - args->postorder_cb = bb_solve_globals; - bb_forward_traversal(args); - } -} - -bool var_check_in_scope(const var_t *var, block_t *block) -{ - const func_t *func = block->func; - - while (block) { - /* Only the first 'size' entries hold a variable; the rest of the - * allocation was never written. Reading them compares against whatever - * the allocator left there, and a stray match puts a variable in scope - * that is not, which changes where phis are inserted and so what code - * comes out -- differently from one build to the next. - */ - for (int i = 0; i < block->locals.size; i++) { - if (var == block->locals.elements[i]) - return true; - } - block = block->parent; - } - - for (int i = 0; i < func->num_params; i++) { - if (&func->param_defs[i] == var) - return true; - } - - if (func->returns_aggregate && &func->sret_def == var) - return true; - - return false; -} - -bool insert_phi_insn(basic_block_t *bb, var_t *var) -{ - bool found = false; - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if ((insn->opcode == OP_phi) && (insn->rd == var)) { - found = true; - break; - } - } - if (found) - return false; - - insn_t *head = bb->insn_list.head; - insn_t *n = arena_calloc(INSN_ARENA, 1, sizeof(insn_t)); - n->opcode = OP_phi; - n->rd = var; - n->rs1 = var; - n->rs2 = var; - if (!head) { - bb->insn_list.head = n; - bb->insn_list.tail = n; - } else { - head->prev = n; - n->next = head; - bb->insn_list.head = n; - } - return true; -} - -void solve_phi_insertion(void) -{ - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - for (symbol_t *sym = func->global_sym_list.head; sym; sym = sym->next) { - var_t *var = sym->var; - - basic_block_t *work_list[PHI_WORKLIST_SIZE]; - int work_list_idx = 0; - - for (ref_block_t *ref = var->ref_block_list.head; ref; - ref = ref->next) { - if (work_list_idx >= PHI_WORKLIST_SIZE - 1) - fatal("PHI worklist overflow"); - work_list[work_list_idx++] = ref->bb; - } - - for (int i = 0; i < work_list_idx; i++) { - basic_block_t *bb = work_list[i]; - for (int j = 0; j < bb->df_idx; j++) { - basic_block_t *df = bb->DF[j]; - if (!var_check_in_scope(var, df->scope)) - continue; - - bool is_decl = false; - for (symbol_t *s = df->symbol_list.head; s; s = s->next) { - if (s->var == var) { - is_decl = true; - break; - } - } - - if (is_decl) - continue; - - if (df == func->exit) - continue; - - /* Neither a global nor an address-taken variable is - * renamed, so there is no version for a phi to choose. - */ - if (var_in_memory(var)) - continue; - - if (insert_phi_insn(df, var)) { - bool found = false; - - /* Restrict phi insertion of ternary operation, and - * logical-and/or operation. - * - * The ternary and logical-and/or operations don't - * create new scope, so prevent temporary variable from - * propagating through the dominance tree. - */ - if (var->is_ternary_ret || var->is_logical_ret) - continue; - - for (int l = 0; l < work_list_idx; l++) { - if (work_list[l] == df) { - found = true; - break; - } - } - if (!found) { - if (work_list_idx >= PHI_WORKLIST_SIZE - 1) - fatal("PHI worklist overflow"); - work_list[work_list_idx++] = df; - } - } - } - } - } - } -} - -var_t *require_var(block_t *blk); - -/* A new variable holding @val, for use as an instruction operand. - * - * It has to be new. rename_var() gives every use reached by one definition the - * same var_t, and mark_const() stamps init_val onto that shared object, so - * rewriting an existing constant's value changes what every other use sees. The - * caller emits the OP_load_constant that defines it. - */ -var_t *new_const_var(block_t *scope, int val) -{ - var_t *var = require_var(scope); - - var->var_name = gen_name(); - var->is_const = true; - var->init_val = val; - var->init_val_hi = 0; - return var; -} -bool is_dominate(const basic_block_t *pred, basic_block_t *succ); - -/* The renaming state of @v, created on first use. */ -rename_t *var_rename(var_t *v) -{ - if (!v->rename) - v->rename = arena_calloc(BLOCK_ARENA, 1, sizeof(rename_t)); - return v->rename; -} - -/* Push a fresh subscript onto @base's renaming stack, growing it as needed. */ -void rename_stack_push(var_t *base, int sub) -{ - rename_t *r = var_rename(base); - if (r->stack_idx >= r->stack_cap) - r->stack = arena_grow(BLOCK_ARENA, (char *) r->stack, &r->stack_cap, - sizeof(int), 8, MAX_RENAME_STACK, - "Too many nested definitions of a variable"); - r->stack[r->stack_idx++] = sub; -} - -/* The one name an address-taken variable keeps for its whole life. - * - * A store through a pointer changes such a variable without any instruction - * naming it, and a read through one sees whatever its last assignment by name - * wrote. Separate SSA versions would give each assignment a place of its own, - * so neither kind of access would reach the other. Like a global, it keeps one - * var_t for every definition and use: the entry version for a parameter, under - * which the allocator finds its incoming register, and the declaration itself - * for anything else. - */ -var_t *addressed_name(var_t *v) -{ - var_t *entry = var_subscript0(v->base); - return entry ? entry : v->base; -} - -void new_name(block_t *block, var_t **var) -{ - var_t *v = *var; - if (!v->base) - v->base = v; - if (v->is_global) - return; - if (v->address_taken) { - var[0] = addressed_name(v); - return; - } - - rename_t *r = var_rename(v->base); - int i = r->counter++; - rename_stack_push(v->base, i); - var_t *vd = require_var(block); - memcpy(vd, *var, sizeof(var_t)); - var_reset_subscripts(vd); /* the copy shares nothing with its base */ - /* A fresh SSA definition has no physical-register residence. In particular, - * it must not inherit either half of a future wide pair from the version it - * was copied from. - */ - vd->phys_reg = -1; - vd->phys_reg_hi = -1; - vd->base = *var; - vd->subscript = i; - var_add_subscript(v, vd); - var[0] = vd; -} - -var_t *get_stack_top_subscript_var(var_t *var) -{ - rename_t *r = var_rename(var->base); - if (r->stack_idx < 1) - return var; /* fallback: use base when no prior definition */ - - int sub = r->stack[r->stack_idx - 1]; - for (int i = 0; i < var->base->subscripts_idx; i++) { - if (var->base->subscripts[i]->subscript == sub) - return var->base->subscripts[i]; - } - - fatal("Failed to find subscript variable on rename stack"); - return NULL; /* unreachable, but silences compiler warning */ -} - -void rename_var(var_t **var) -{ - var_t *v = *var; - if (!v->base) - v->base = v; - if (v->is_global) - return; - if (v->address_taken) { - var[0] = addressed_name(v); - return; - } - - var[0] = get_stack_top_subscript_var(*var); -} - -void pop_name(var_t *var) -{ - if (var_in_memory(var)) - return; - - /* Pop unconditionally, creating the state if the variable has none: the - * inline rename_t this replaced was always present, so a pop with nothing - * pushed drove stack_idx to -1, and the next push then landed one slot - * below the stack. Preserve that exactly. - */ - rename_t *r = var_rename(var->base); - r->stack_idx--; -} - -void append_phi_operand(insn_t *insn, var_t *var, basic_block_t *bb_from) -{ - phi_operand_t *op = arena_calloc(GENERAL_ARENA, 1, sizeof(phi_operand_t)); - op->from = bb_from; - op->var = get_stack_top_subscript_var(var); - - phi_operand_t *tail = insn->phi_ops; - if (tail) { - while (tail->next) - tail = tail->next; - tail->next = op; - } else - insn->phi_ops = op; -} - -void bb_solve_phi_params(basic_block_t *bb) -{ - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->opcode == OP_phi) - new_name(bb->scope, &insn->rd); - else { - if (insn->rs1) - rename_var(&insn->rs1); - if (insn->rs2) - if (!insn->rs2->is_func) - rename_var(&insn->rs2); - if (insn->rd) - new_name(bb->scope, &insn->rd); - } - } - - if (bb->next) { - for (insn_t *insn = bb->next->insn_list.head; insn; insn = insn->next) { - if (insn->opcode == OP_phi) - append_phi_operand(insn, insn->rd, bb); - } - } - - if (bb->then_) { - for (insn_t *insn = bb->then_->insn_list.head; insn; - insn = insn->next) { - if (insn->opcode == OP_phi) - append_phi_operand(insn, insn->rd, bb); - } - } - - if (bb->else_) { - for (insn_t *insn = bb->else_->insn_list.head; insn; - insn = insn->next) { - if (insn->opcode == OP_phi) - append_phi_operand(insn, insn->rd, bb); - } - } - - for (int i = 0; i < bb->dom_next_idx; i++) - bb_solve_phi_params(bb->dom_next[i]); - - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->opcode == OP_phi) - pop_name(insn->rd); - else if (insn->rd) - pop_name(insn->rd); - } -} - -void solve_phi_params(void) -{ - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - /* The aggregate return destination is an ABI parameter even though it - * is not a source-language parameter. Build its entry SSA version - * before the visible parameters. - */ - for (int i = -1; i < func->num_params; i++) { - /* FIXME: Direct argument renaming in SSA construction phase may - * interfere with later optimization passes - */ - var_t *var = require_var(func->bbs->scope); - if (i < 0 && !func->returns_aggregate) - continue; - var_t *base = i < 0 ? &func->sret_def : &func->param_defs[i]; - memcpy(var, base, sizeof(var_t)); - var_reset_subscripts(var); /* the copy shares nothing with base */ - var->phys_reg = -1; - var->phys_reg_hi = -1; - var->base = base; - var->subscript = 0; - - rename_t *r = var_rename(base); - rename_stack_push(base, r->counter++); - var_add_subscript(base, var); - } - - bb_solve_phi_params(func->bbs); - } -} - -void append_unwound_phi_insn(basic_block_t *bb, var_t *dest, var_t *rs) -{ - insn_t *n = arena_calloc(INSN_ARENA, 1, sizeof(insn_t)); - n->opcode = OP_unwound_phi; - n->rd = dest; - n->rs1 = rs; - n->belong_to = bb; - - insn_t *tail = bb->insn_list.tail; - if (!tail) { - bb->insn_list.head = n; - bb->insn_list.tail = n; - } else { - /* insert it before branch instruction */ - if (tail->opcode == OP_branch) { - if (tail->prev) { - tail->prev->next = n; - n->prev = tail->prev; - } else - bb->insn_list.head = n; - - n->next = tail; - tail->prev = n; - } else { - tail->next = n; - bb->insn_list.tail = n; - } - } -} - -void bb_unwind_phi(func_t *func, basic_block_t *bb) -{ - UNUSED(func); - - insn_t *insn; - for (insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->opcode != OP_phi) - break; - - for (phi_operand_t *operand = insn->phi_ops; operand; - operand = operand->next) - append_unwound_phi_insn(operand->from, insn->rd, operand->var); - } - - bb->insn_list.head = insn; - if (!insn) - bb->insn_list.tail = NULL; - else - insn->prev = NULL; -} - -int loop_scan_gen; - -/* Raise the depth of every block in the natural loop that the edge from @latch - * back to @header closes: the header, and everything that can reach the latch - * without leaving the loop. - */ -bool mark_natural_loop(basic_block_t *header, basic_block_t *latch) -{ - basic_block_t *stack[MAX_LOOP_WALK]; - int sp = 0; - - loop_scan_gen++; - header->loop_mark = loop_scan_gen; - header->loop_depth++; - - if (latch != header) { - latch->loop_mark = loop_scan_gen; - latch->loop_depth++; - stack[sp++] = latch; - } - - while (sp) { - basic_block_t *bb = stack[--sp]; - - for (int i = 0; i < bb->prev_idx; i++) { - basic_block_t *p = bb->prev[i].bb; - - if (!p || p->loop_mark == loop_scan_gen) - continue; - - /* Out of room to widen. Marking this block and then not walking - * through it would leave the rest of the loop unmarked while - * looking marked, and a reader of loop_mark would take blocks - * inside the loop for blocks outside it. Report the whole answer as - * unusable instead. - */ - if (sp >= MAX_LOOP_WALK) - return false; - p->loop_mark = loop_scan_gen; - p->loop_depth++; - stack[sp++] = p; - } - } - return true; -} - -/* Count the loops enclosing each block of @func. - * - * A loop is closed by an edge to a block that dominates its source. Position - * along the rpo_next chain cannot stand in for that: the passes that splice - * blocks out of the chain leave an if's arms sitting after the block they - * rejoin, and every one of those edges would read as a loop. - */ -void mark_loop_depth(func_t *func) -{ - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - bb->loop_depth = 0; - bb->loop_mark = 0; - } - - - for (basic_block_t *p = func->bbs; p; p = p->rpo_next) { - basic_block_t *succ[3]; - - succ[0] = p->next; - succ[1] = p->then_; - succ[2] = p->else_; - - for (int k = 0; k < 3; k++) { - if (!succ[k]) - continue; - if (succ[k] != p && !is_dominate(succ[k], p)) - continue; - - /* A walk that ran out of room stopped partway, leaving the depths - * it had already raised standing over a region it never finished - * measuring. Those feed the weights pin_registers() compares, so a - * half-counted loop would outbid a whole one; drop the tally for - * the function instead and let every block weigh the same. - */ - if (!mark_natural_loop(succ[k], p)) { - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - bb->loop_depth = 0; - bb->loop_weight = 1; - } - return; - } - } - } - - /* What one naming in each block is worth, so that the tally in - * pin_registers() reads it rather than deriving it per operand. - */ - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - int w = 1; - - for (int d = 0; d < bb->loop_depth && d < MAX_WEIGHTED_LOOP_DEPTH; d++) - w = w * LOOP_USE_WEIGHT; - bb->loop_weight = w; - } -} - -/* Put @insn into @bb straight after @after, or at the head when it is NULL. */ -void bb_insert_after(basic_block_t *bb, insn_t *after, insn_t *insn) -{ - insn->belong_to = bb; - insn->prev = after; - if (after) { - insn->next = after->next; - after->next = insn; - } else { - insn->next = bb->insn_list.head; - bb->insn_list.head = insn; - } - if (insn->next) - insn->next->prev = insn; - else - bb->insn_list.tail = insn; -} - -/* Append @insn to @bb, detaching it from wherever it was. */ -void bb_append_insn(basic_block_t *bb, insn_t *insn) -{ - bb_insert_after(bb, bb->insn_list.tail, insn); -} - -/* A fresh instruction naming @rd, @rs1 and @rs2, not yet in any block. The - * allocation is zeroed, so every other field starts empty -- which is what the - * passes that build instructions want, and one less thing for each of them to - * remember. - */ -insn_t *new_insn(opcode_t op, var_t *rd, var_t *rs1, var_t *rs2) -{ - insn_t *insn = arena_calloc(INSN_ARENA, 1, sizeof(insn_t)); - - insn->opcode = op; - insn->rd = rd; - insn->rs1 = rs1; - insn->rs2 = rs2; - return insn; -} - -/* Whether @insn computes a value with no side effect and no way to fault, so - * running it on a path that would not have reached it changes nothing. - */ -bool insn_is_speculatable(const insn_t *insn) -{ - /* Running an arm unconditionally would read a volatile object on a path - * that never evaluates it, and that read is a side effect. - */ - if (insn_reads_volatile(insn)) - return false; - - switch (insn->opcode) { - case OP_add: - case OP_sub: - case OP_mul: - case OP_lshift: - case OP_rshift: - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: - case OP_negate: - case OP_bit_not: - case OP_log_not: - case OP_eq: - case OP_neq: - case OP_lt: - case OP_leq: - case OP_gt: - case OP_geq: - case OP_assign: - case OP_load_constant: - return true; - default: - /* Division can trap, a read can fault, a write or a call is a side - * effect, and anything else is not worth reasoning about here. - */ - return false; - } -} - -/* Walk the arm starting at @arm, gathering the blocks it runs before control - * rejoins. - * - * An arm is not one block: the copy carrying its value to the join is appended - * to whichever block immediately precedes the join, which is separate from the - * one holding the arm's computation. The walk follows single-entry, single-exit - * blocks and stops at the first block something else can also reach, which is - * the join. - */ -basic_block_t *if_arm_chain(basic_block_t *arm, - basic_block_t **chain, - int *len, - int *count) -{ - basic_block_t *bb = arm; - - *len = 0; - *count = 0; - - while (bb) { - if (bb->then_ || bb->else_ || !bb->next) - return NULL; - if (bb_pred_count(bb) != 1) - return NULL; - if (*len >= MAX_IF_ARM_BLOCKS) - return NULL; - - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->opcode == OP_unwound_phi) - continue; - if (!insn_is_speculatable(insn)) - return NULL; - - /* Writing a global or an address-taken variable is a store, and a - * store is a side effect however plain the arithmetic producing it - * looks. - */ - if (insn->rd && var_in_memory(insn->rd)) - return NULL; - *count = *count + 1; - } - - chain[*len] = bb; - *len = *len + 1; - - if (bb_pred_count(bb->next) != 1) - return bb->next; - bb = bb->next; - } - return NULL; -} - -/* The single copy an arm hands to the join, or NULL when it hands over more or - * fewer than one. - */ -insn_t *if_chain_phi(basic_block_t **chain, int len) -{ - insn_t *phi = NULL; - - for (int i = 0; i < len; i++) { - for (insn_t *insn = chain[i]->insn_list.head; insn; insn = insn->next) { - if (insn->opcode != OP_unwound_phi) - continue; - if (phi) - return NULL; - phi = insn; - } - } - return phi; -} - -/* Whether any instruction in @b reads a value defined in @a. */ -bool if_arm_reads_other(basic_block_t **a, - int alen, - basic_block_t **b, - int blen) -{ - for (int i = 0; i < alen; i++) { - for (insn_t *def = a[i]->insn_list.head; def; def = def->next) { - if (!def->rd) - continue; - for (int j = 0; j < blen; j++) { - for (insn_t *use = b[j]->insn_list.head; use; use = use->next) { - if (use->rs1 == def->rd || use->rs2 == def->rd || - use->rs3 == def->rd) - return true; - } - } - } - } - return false; -} - -/* Whether @var is an integer scalar a 32-bit target keeps in a register pair. - * The widening pass below, the allocator and the backends all ask this. - */ -bool var_needs_register_pair(const var_t *var) -{ - return PTR_SIZE < 8 && var && !var->ptr_level && !var->is_func && - !var->array_size && var->type && var->type->size == 8 && - !is_record_type(var->type); -} - -/* Flatten "if (c) x = A; else x = B;" into both computations and a select. - * - * A branch the hardware cannot predict costs far more than the arm it skips: - * the CRC loop in the benchmark suite mispredicts 17% of 283 million branches - * and pays about a thousand million cycles for it, most of its run time. - * Running both arms and selecting between the results has nothing to - * mispredict, and neither arm may write memory, call anything or divide, so - * running the one that would have been skipped costs only itself. - */ -bool if_convert_bb(func_t *func, basic_block_t *bb) -{ - basic_block_t *t = bb->then_; - basic_block_t *e = bb->else_; - basic_block_t *t_chain[MAX_IF_ARM_BLOCKS]; - basic_block_t *e_chain[MAX_IF_ARM_BLOCKS]; - int t_len = 0; - int e_len = 0; - int tn = 0; - int en = 0; - - if (!t || !e || t == e) - return false; - - basic_block_t *join = if_arm_chain(t, t_chain, &t_len, &tn); - basic_block_t *e_join = if_arm_chain(e, e_chain, &e_len, &en); - - if (!join || join != e_join) - return false; - - /* Flattening removes both arms, so the join must be reached from them and - * nothing else. - */ - if (bb_pred_count(join) != 2) - return false; - - insn_t *t_phi = if_chain_phi(t_chain, t_len); - insn_t *e_phi = if_chain_phi(e_chain, e_len); - if (!t_phi || !e_phi || t_phi->rd != e_phi->rd) - return false; - if (var_in_memory(t_phi->rd)) - return false; - /* Both arms handing over the same value makes the select pointless. */ - if (t_phi->rs1 == e_phi->rs1) - return false; - if (tn + en > MAX_SPECULATED_INSNS) - return false; - - insn_t *br = bb->insn_list.tail; - if (!br || br->opcode != OP_branch || !br->rs1) - return false; - - /* The arms run one after the other now, so whichever computes a value the - * other reads has to go first -- common subexpression elimination leaves - * exactly that. If each reads something the other defines, no order works. - */ - bool e_needs_t = if_arm_reads_other(t_chain, t_len, e_chain, e_len); - bool t_needs_e = if_arm_reads_other(e_chain, e_len, t_chain, t_len); - - if (e_needs_t && t_needs_e) - return false; - - var_t *cond = br->rs1; - - bb->insn_list.tail = br->prev; - if (br->prev) - br->prev->next = NULL; - else - bb->insn_list.head = NULL; - - insn_t *next; - - for (int step = 0; step < 2; step++) { - bool take_t = e_needs_t ? step == 0 : step == 1; - basic_block_t **chain = take_t ? t_chain : e_chain; - const insn_t *skip = take_t ? t_phi : e_phi; - int len = take_t ? t_len : e_len; - - for (int i = 0; i < len; i++) { - for (insn_t *insn = chain[i]->insn_list.head; insn; insn = next) { - next = insn->next; - if (insn == skip) - continue; - - /* Whatever the arm computes is a version of the variable the - * select writes as often as not, and that variable's pinned - * register still has to carry the value the arms read. - */ - if (insn->rd) - insn->rd->in_select_arm = true; - bb_append_insn(bb, insn); - } - chain[i]->insn_list.head = NULL; - chain[i]->insn_list.tail = NULL; - } - } - - /* One instruction naming all three inputs: split into a copy and a - * read-modify move, the allocator settles their registers separately and - * need not agree on one. - */ - t_phi->opcode = OP_cmov; - t_phi->rs2 = cond; - t_phi->rs3 = e_phi->rs1; - bb_append_insn(bb, t_phi); - - /* Rewire: the test now falls straight into the join. */ - bb_disconnect(t_chain[t_len - 1], join); - bb_disconnect(e_chain[e_len - 1], join); - bb_disconnect(bb, t); - bb_disconnect(bb, e); - bb_connect(bb, join, NEXT); - - /* Take the arms out of the order the rest of the compiler walks. */ - for (basic_block_t *p = func->bbs; p; p = p->rpo_next) { - bool drop = true; - - while (drop && p->rpo_next) { - drop = false; - for (int i = 0; i < t_len; i++) { - if (p->rpo_next == t_chain[i]) - drop = true; - } - for (int i = 0; i < e_len; i++) { - if (p->rpo_next == e_chain[i]) - drop = true; - } - if (drop) - p->rpo_next = p->rpo_next->rpo_next; - } - } - return true; -} - -/* The generation stamping the loop strength_reduce() is looking at, so that - * var_read_by() can be asked about that loop alone. - */ -int sr_gen; - -/* Whether anything in @func reads @var, ignoring the @nskip instructions in - * @skip. - * - * With @loop_only set, only the blocks the stamped loop covers are searched: a - * reader outside it does not keep a literal alive inside, because the allocator - * materialises one wherever the value is wanted. - */ -bool var_read_by(func_t *func, - const var_t *var, - insn_t **skip, - int nskip, - bool loop_only) -{ - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - if (loop_only && bb->loop_mark != sr_gen) - continue; - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - bool skipped = false; - - for (int i = 0; i < nskip; i++) { - if (skip[i] == insn) - skipped = true; - } - if (skipped) - continue; - if (insn->rs1 == var || insn->rs2 == var || insn->rs3 == var) - return true; - } - } - return false; -} - -/* Detach @insn from the block holding it. */ -void bb_remove_insn(basic_block_t *bb, insn_t *insn) -{ - if (insn->prev) - insn->prev->next = insn->next; - else - bb->insn_list.head = insn->next; - if (insn->next) - insn->next->prev = insn->prev; - else - bb->insn_list.tail = insn->prev; - insn->prev = NULL; - insn->next = NULL; -} - -/* Send the arms of "if (a && b)" straight where they are going. - * - * A short-circuit condition becomes a block per arm, each writing 0 or 1 into - * one temporary, and a join that does nothing but branch on it. The value is - * born a constant and dies at that branch, so the arm already knows which way - * it goes: writing the answer to a stack slot, reading it back and testing it - * is five instructions to reach a label the arm could have jumped to. Every - * loop guarded by "i < n && p[i]" pays them on each iteration. - */ -bool thread_const_branch(func_t *func, basic_block_t *join) -{ - insn_t *br = join->insn_list.head; - - if (!join->then_ || !join->else_) - return false; - - /* Only a block that does nothing else: anything before the branch would be - * skipped, and a phi copy left in it belongs to a successor. - */ - if (!br || br->next || br->opcode != OP_branch || !br->rs1) - return false; - - var_t *cond = br->rs1; - bool threaded = false; - - /* Two passes: the first only looks, because the reader scan between them - * walks the whole function and is not worth paying for a join no - * predecessor can be threaded to anyway. - */ - for (int pass = 0; pass < 2; pass++) { - for (int i = 0; i < join->prev_idx; i++) { - basic_block_t *pred = join->prev[i].bb; - - if (!pred || pred == join) - continue; - - /* A predecessor that also goes somewhere else keeps a branch of its - * own, and rewiring one of its edges would need that branch - * rewritten. - */ - if (pred->then_ || pred->else_ || pred->next != join) - continue; - - insn_t *def = pred->insn_list.tail; - - if (!def || def->rd != cond) - continue; - if (def->opcode != OP_unwound_phi && def->opcode != OP_assign) - continue; - if (!def->rs1 || !def->rs1->is_const) - continue; - - if (!pass) { - threaded = true; - break; - } - - /* A wide constant with a zero low word, such as 1ULL << 32, is - * still true. - */ - basic_block_t *target = def->rs1->init_val || def->rs1->init_val_hi - ? join->then_ - : join->else_; - - bb_remove_insn(pred, def); - bb_disconnect(pred, join); - bb_connect(pred, target, NEXT); - } - - if (!threaded) - return false; - if (!pass) { - /* The condition must die here: threading removes the writes, so - * anything else reading it would read nothing. - */ - insn_t *skip[1]; - - skip[0] = br; - if (var_read_by(func, cond, skip, 1, false)) - return false; - } - } - - /* Nothing reaches the join any more, so take it out of the order the rest - * of the compiler walks -- and with it the branch, whose condition no - * longer has a definition on any path. - */ - if (!bb_pred_count(join)) { - bb_disconnect(join, join->then_); - bb_disconnect(join, join->else_); - for (basic_block_t *p = func->bbs; p && p->rpo_next; p = p->rpo_next) { - if (p->rpo_next == join) { - p->rpo_next = join->rpo_next; - break; - } - } - } - return true; -} - -void thread_const_branches(void) -{ - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - if (!func->bbs) - continue; - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - const insn_t *br = bb->insn_list.head; - - if (!br || br->next || br->opcode != OP_branch || !br->rs1) - continue; - thread_const_branch(func, bb); - } - } -} - -/* Copy a small function into its callers. - * - * A call to a four-line helper costs more than the helper: the arguments go - * into the argument registers, everything live crosses a call boundary and so - * goes to the frame, and the callee builds and tears down a frame of its own. - * The benchmark suite's "calls" case spends three instructions on that for - * every one it spends computing. Copying the body in removes all of it and lets - * the optimizer see the caller and the callee together. - * - * Only a body with no control flow of its own is copied, which keeps the - * transformation to splicing one instruction list into another. - */ -var_t *inline_from[MAX_INLINE_VARS]; -var_t *inline_to[MAX_INLINE_VARS]; -int inline_map_n; - -/* The caller's stand-in for the callee's @var: the argument for a parameter, a - * fresh variable for anything the body computes, and the variable itself for - * anything shared. - */ -var_t *inline_lookup(var_t *var, block_t *scope) -{ - /* A global is shared, not copied. */ - if (!var || var->is_global) - return var; - - for (int i = 0; i < inline_map_n; i++) { - if (inline_from[i] == var) - return inline_to[i]; - - /* Parameters are mapped by the variable they are versions of, so that a - * body naming a later version still finds the argument. - */ - if (inline_from[i] && inline_from[i] == var->base) - return inline_to[i]; - } - - if (inline_map_n >= MAX_INLINE_VARS) - return NULL; - - var_t *copy = require_var(scope); - - copy->var_name = gen_name(); - copy->type = var->type; - copy->ptr_level = var->ptr_level; - copy->is_const = var->is_const; - copy->init_val = var->init_val; - copy->init_val_hi = var->init_val_hi; - inline_from[inline_map_n] = var; - inline_to[inline_map_n] = copy; - inline_map_n++; - return copy; -} - -/* Whether @func is small and straight-line enough to copy into its callers. - * - * A function is a chain of blocks even when its source has no control flow: the - * entry block holds the declarations and falls into the body. Any block that - * branches, or that something else can reach, ends the chain and makes the - * function too complicated to splice into an instruction list. - */ -int inline_round; - -/* Whether cloning @func's body would need more renaming entries than the map - * holds. - * - * inline_lookup() returns NULL once it is full, and inline_call_at() has - * already removed the push/call/retval sequence by the time cloning starts, so - * a failure there cannot be declined -- it leaves instructions carrying null - * operands behind, which the register allocator dereferences. Counting before - * anything is touched is what keeps that from needing to be undone. - * - * The count mirrors inline_lookup()'s matching exactly: a global is shared - * rather than copied, a parameter is found through the variable it is a version - * of, and every other version gets an entry of its own. - */ -bool inline_map_fits(func_t *func) -{ - var_t *seen[MAX_INLINE_VARS]; - int n = 0; - - for (int q = 0; q < func->num_params; q++) { - if (n >= MAX_INLINE_VARS) - return false; - seen[n++] = &func->param_defs[q]; - } - - for (basic_block_t *bb = func->bbs; bb; bb = bb->next) { - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - var_t *ops[4]; - - ops[0] = insn->rd; - ops[1] = insn->rs1; - ops[2] = insn->rs2; - ops[3] = insn->rs3; - - for (int k = 0; k < 4; k++) { - var_t *var = ops[k]; - - if (!var || var->is_global) - continue; - - bool found = false; - - for (int i = 0; i < n; i++) { - if (seen[i] == var || seen[i] == var->base) - found = true; - } - if (found) - continue; - if (n >= MAX_INLINE_VARS) - return false; - seen[n++] = var; - } - } - } - return true; -} - -bool func_is_inlinable(func_t *func) -{ - int n = 0, blocks = 0; - insn_t *last = NULL; - - /* The answer only changes when a round changes the body. */ - if (func->inline_gen == inline_round) - return func->inline_ok; - func->inline_gen = inline_round; - func->inline_ok = false; - func->inline_ret = NULL; - - if (!func->bbs || func->va_args) - return false; - if (func->num_params > MAX_ARGS_IN_REG) - return false; - - for (basic_block_t *bb = func->bbs; bb; bb = bb->next) { - if (bb->then_ || bb->else_) - return false; - if (bb != func->bbs && bb_pred_count(bb) != 1) - return false; - - /* The single-predecessor test above is skipped for the entry block, so - * a "next" chain that came back round to it would walk for ever. A - * chain longer than the instruction budget cannot be inlinable in any - * case, so stopping there costs nothing and bounds the walk. - */ - if (++blocks > MAX_INLINE_INSNS) - return false; - - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - n++; - if (n > MAX_INLINE_INSNS) - return false; - last = insn; - if (insn->opcode == OP_return) - continue; - - /* A call of its own would have to be copied as a call, which is - * what the copying is meant to remove; the pass runs again once - * that callee has been copied in, and by then this body qualifies. - */ - if (!insn_is_speculatable(insn)) - return false; - if (insn->rd && var_in_memory(insn->rd)) - return false; - - /* Writing a parameter would mean the copy assigns to the caller's - * own variable, since a parameter maps onto the argument. - */ - for (int q = 0; q < func->num_params; q++) { - if (insn->rd && insn->rd->base == &func->param_defs[q]) - return false; - } - } - } - - if (!last || last->opcode != OP_return) - return false; - if (!inline_map_fits(func)) - return false; - - func->inline_ok = true; - func->inline_ret = last; - return true; -} - -/* Copy @src into @bb after @after, renaming what it names. */ -insn_t *inline_clone(basic_block_t *bb, - insn_t *after, - insn_t *src, - block_t *scope) -{ - insn_t *copy = new_insn(src->opcode, inline_lookup(src->rd, scope), - inline_lookup(src->rs1, scope), - inline_lookup(src->rs2, scope)); - - copy->rs3 = inline_lookup(src->rs3, scope); - copy->sz = src->sz; - copy->str = src->str; - bb_insert_after(bb, after, copy); - return copy; -} - -/* Replace the push/call/retval sequence ending at @call with the callee's body. - * - * Returns the instruction to carry on scanning from, which is NULL when the - * copy lands at the end of the block; *@done says whether the call was replaced - * at all. The two have to be reported separately -- a NULL return read as "left - * alone" stopped the round from being counted as progress and left the rest of - * the block unscanned. - * - * @done is an int and not a bool on purpose. A _Bool is one byte, and writing - * one byte through a pointer into the caller's slot -- which is pointer-sized - * -- leaves the bytes above it holding whatever was there before, so the read - * back can be true when false was written. Compiling shecc with itself for - * RISC-V is where that shows: the copy of the compiler so built inlines less - * than the one gcc built, and stage1 and stage2 stop matching. Widening the - * flag sidesteps it; the underlying narrow-store bug is still there for any - * other "bool *" to find. - */ -insn_t *inline_call_at(basic_block_t *bb, insn_t *call, int *done) -{ - func_t *callee = find_func(call->str); - - *done = 0; - if (!callee || !func_is_inlinable(callee)) - return NULL; - - /* The arguments are the pushes immediately before the call. */ - var_t *args[MAX_PARAMS]; - insn_t *first = call; - int argc = 0; - - for (insn_t *p = call->prev; p && p->opcode == OP_push; p = p->prev) { - first = p; - argc++; - } - if (argc != callee->num_params || argc > MAX_PARAMS) - return NULL; - - /* A call nested in another call's argument list -- "f(g(x), y)" -- has the - * outer call's pushes already standing before it. The register allocator - * hands those the argument registers as it meets them, so anything spliced - * in between would overwrite arguments the outer call is still waiting to - * make. - */ - for (insn_t *p = first->prev; p; p = p->prev) { - if (p->opcode == OP_call || p->opcode == OP_indirect) - break; - if (p->opcode == OP_push) - return NULL; - } - - insn_t *walk = first; - for (int i = 0; i < argc; i++) { - args[i] = walk->rs1; - walk = walk->next; - } - - insn_t *last = call; - var_t *dest = NULL; - - if (call->next && call->next->opcode == OP_func_ret) { - last = call->next; - dest = last->rd; - } - - insn_t *ret = callee->inline_ret; - - /* A value the caller wants and the callee does not produce, or the other - * way round, would need a fabricated definition. - */ - if (!dest != !ret->rs1) - return NULL; - - inline_map_n = 0; - for (int i = 0; i < argc; i++) { - if (inline_map_n >= MAX_INLINE_VARS) - return NULL; - inline_from[inline_map_n] = &callee->param_defs[i]; - inline_to[inline_map_n] = args[i]; - inline_map_n++; - } - - block_t *scope = bb->scope; - insn_t *after = first->prev; - insn_t *next; - - for (insn_t *dead = first; dead; dead = next) { - next = dead->next; - bb_remove_insn(bb, dead); - if (dead == last) - break; - } - - for (basic_block_t *src_bb = callee->bbs; src_bb; src_bb = src_bb->next) { - for (insn_t *src = src_bb->insn_list.head; src; src = src->next) { - if (src == ret) - break; - after = inline_clone(bb, after, src, scope); - } - } - - if (dest) { - insn_t *copy = - new_insn(OP_assign, dest, inline_lookup(ret->rs1, scope), NULL); - - bb_insert_after(bb, after, copy); - after = copy; - } - - *done = 1; - if (after) - return after->next; - return bb->insn_list.head; -} - -void inline_calls(void) -{ - for (int round = 0; round < MAX_INLINE_ROUNDS; round++) { - bool changed = false; - - inline_round++; - - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - if (!func->bbs) - continue; - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - insn_t *next; - - for (insn_t *insn = bb->insn_list.head; insn; insn = next) { - next = insn->next; - if (insn->opcode != OP_call || !insn->str) - continue; - - int done; - insn_t *resume = inline_call_at(bb, insn, &done); - - if (done) { - next = resume; - changed = true; - - /* The body just grew, so the verdict cached for it this - * round no longer describes it -- and losing a call may - * be exactly what makes it copyable. - */ - func->inline_gen = 0; - } - } - } - } - if (!changed) - break; - } -} - -/* Walk an array with a pointer instead of recomputing its address. - * - * "a[i * 64 + k]" inside the k loop spends four instructions turning k into an - * address, every one of which changes by the same amount from one iteration to - * the next. Computing the first address before the loop and adding that amount - * at the bottom leaves the body with the access alone -- matmul's innermost - * loop spent six of its sixteen instructions on the two subscripts. - * - * Unlike hoisting a loop-invariant value, this pays even when the result has to - * live on the frame: the loop trades a four-instruction recomputation for one - * addition, where hoisting trades a recomputation for a reload. The variable - * the loop counts with, where the chain walk stops: its value before the loop - * is what the first address is computed from. - */ -var_t *sr_iv; - -var_t *sr_base(var_t *var) -{ - if (!var) - return NULL; - return var->base ? var->base : var; -} - -/* Whether @var is written inside the loop being examined. A constant never is, - * wherever its materialisation sits: the allocator emits one wherever the value - * is wanted. - */ -bool sr_varies(var_t *var) -{ - const var_t *base; - - if (!var || var->is_const) - return false; - base = sr_base(var); - return base && base->loop_stamp == sr_gen; -} - -/* How much @var moves per iteration, into *step. Reports false when that is not - * known -- which for a variable the loop writes means "not yet derived". - */ -bool sr_step_of(var_t *var, int *step) -{ - const var_t *base; - - if (!var) - return false; - if (var->is_const) { - *step = 0; - return true; - } - base = sr_base(var); - if (!base) - return false; - if (base->iv_gen == sr_gen) { - *step = base->iv_step; - return true; - } - if (base->loop_stamp != sr_gen) { - *step = 0; /* nothing in the loop writes it */ - return true; - } - return false; -} - -void sr_set_step(var_t *var, int step) -{ - var_t *base = sr_base(var); - - if (!base) - return; - base->iv_gen = sr_gen; - base->iv_step = step; -} - -/* Whether @insn is one the pass is willing to lift out of a loop: it computes a - * value from its operands, touches no memory and faults on nothing. Taking an - * array's address qualifies -- that address is the same on every iteration -- - * and a literal does not, because the allocator materialises one wherever it is - * wanted and moving it would only lengthen a live range. - */ -bool sr_movable(const insn_t *insn) -{ - switch (insn->opcode) { - case OP_add: - case OP_sub: - case OP_mul: - case OP_lshift: - case OP_rshift: - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: - case OP_assign: - case OP_address_of: - case OP_global_address_of: - case OP_load_data_address: - case OP_load_rodata_address: - return true; - default: - return false; - } -} - -/* The instruction inside the loop that defines @var, or NULL. */ -insn_t *sr_def_of(func_t *func, const var_t *var) -{ - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - if (bb->loop_mark != sr_gen) - continue; - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->rd == var) - return insn; - } - } - return NULL; -} - -/* Find the variable the loop counts with, and how far it counts each time. - * - * It is the one whose only write in the loop adds a literal to itself, which - * after phi unwinding is a pair: a temporary holding the sum, and a copy of - * that temporary back into the variable. - */ -var_t *sr_basic_iv(func_t *func, basic_block_t *latch, int *step) -{ - var_t *found = NULL; - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - if (bb->loop_mark != sr_gen) - continue; - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->opcode != OP_assign && insn->opcode != OP_unwound_phi) - continue; - if (!insn->rd || !insn->rs1) - continue; - - var_t *k = sr_base(insn->rd); - insn_t *sum = sr_def_of(func, insn->rs1); - - if (!k || !sum) - continue; - if (sum->opcode != OP_add && sum->opcode != OP_sub) - continue; - if (!sum->rs1 || !sum->rs2) - continue; - - const var_t *addend = NULL; - var_t *carried = NULL; - int sign = 1; - - if (sr_base(sum->rs1) == k && sum->rs2->is_const) { - addend = sum->rs2; - carried = sum->rs1; - if (sum->opcode == OP_sub) - sign = -1; /* counting down */ - } else if (sum->opcode == OP_add && sr_base(sum->rs2) == k && - sum->rs1->is_const) { - addend = sum->rs1; - carried = sum->rs2; - } - if (!addend || !addend->init_val) - continue; - - /* The literal has to be added to the variable itself, not to - * something computed from it: a temporary made from a variable - * carries that variable as its base, so "g = g * 2 + 1" matches the - * shape as readily as "i = i + 1" and would be taken for a counter - * stepping by one. The value flowing in is the one the loop either - * carries round in a phi or never writes. - */ - insn_t *src = sr_def_of(func, carried); - - if (src) { - if (src->opcode != OP_assign && src->opcode != OP_unwound_phi) - continue; - if (sr_base(src->rs1) != k) - continue; - } - /* Two counters would each need their own analysis. */ - if (found && found != k) - return NULL; - found = k; - *step = sign * addend->init_val; - } - } - if (!found) - return NULL; - - /* One variable the loop writes twice is not a counter: "i++" in one arm and - * "i--" in another moves it by neither step, and an address derived from it - * would advance by a fixed amount that matches neither. - */ - int writes = 0; - basic_block_t *wblk = NULL; - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - if (bb->loop_mark != sr_gen) - continue; - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - /* Only a copy back into the variable counts. The arithmetic that - * produces the new value writes a temporary, and a temporary made - * from a variable carries that variable as its base. Only a write - * of the variable itself counts. The arithmetic that produces the - * new value writes a temporary, and a temporary made from a - * variable carries that variable as its base -- but a select names - * the variable, and one writing the counter means the counter does - * not advance on every trip round. - */ - if (insn->opcode != OP_assign && insn->opcode != OP_unwound_phi && - insn->opcode != OP_cmov) - continue; - if (!insn->rd || sr_base(insn->rd) != found) - continue; - - /* A phi copying the variable to itself carries it around the loop - * rather than giving it a new value. - */ - if (insn->opcode != OP_cmov && sr_base(insn->rs1) == found) - continue; - writes++; - wblk = bb; - } - } - if (writes != 1) - return NULL; - - /* The write has to happen on every trip round, because the pointer derived - * from the counter is advanced on every trip round. A "continue" that jumps - * over the sole increment leaves the two disagreeing, and the loop reads - * one element too far from then on. - */ - if (wblk != latch && !is_dominate(wblk, latch)) - return NULL; - return found; -} - -/* Derive how far every value in the loop moves, from the counter outwards. - * Several rounds because a chain is only as derivable as its operands. - */ -void sr_derive_steps(func_t *func) -{ - for (int round = 0; round < MAX_IV_ROUNDS; round++) { - bool changed = false; - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - if (bb->loop_mark != sr_gen) - continue; - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - int a = 0, b = 0, s = 0; - - if (!insn->rd) - continue; - - const var_t *base = sr_base(insn->rd); - - if (!base || base->iv_gen == sr_gen) - continue; - if (base->def_cnt != 1) - continue; - if (!sr_step_of(insn->rs1, &a)) - continue; - - switch (insn->opcode) { - case OP_assign: - s = a; - break; - case OP_add: - if (!sr_step_of(insn->rs2, &b)) - continue; - s = a + b; - break; - case OP_sub: - if (!sr_step_of(insn->rs2, &b)) - continue; - s = a - b; - break; - case OP_mul: - if (!insn->rs2 || !insn->rs2->is_const) - continue; - s = a * insn->rs2->init_val; - break; - case OP_lshift: - if (!insn->rs2 || !insn->rs2->is_const) - continue; - if (insn->rs2->init_val < 0 || insn->rs2->init_val > 30) - continue; - s = a << insn->rs2->init_val; - break; - default: - continue; - } - sr_set_step(insn->rd, s); - changed = true; - } - } - if (!changed) - return; - } -} - -/* Gather, into @chain, the instructions of the loop that compute @var, in the - * order they appear. Reports false when any of them is shared, has a side - * effect, or the chain is longer than the pass is willing to move. - */ -bool sr_collect_chain(func_t *func, var_t *var, insn_t **chain, int *len) -{ - if (sr_base(var) == sr_iv) - return true; /* the counter itself: the preheader has its first value */ - - insn_t *def = sr_def_of(func, var); - - if (!def) - return false; - for (int i = 0; i < *len; i++) { - if (chain[i] == def) - return true; /* already collected */ - } - if (*len >= MAX_IV_CHAIN) - return false; - if (!sr_movable(def)) - return false; - - const var_t *base = sr_base(def->rd); - - if (!base || base->def_cnt != 1) - return false; - if (var_in_memory(base)) - return false; - - chain[*len] = def; - *len = *len + 1; - - var_t *ops[3]; - - ops[0] = def->rs1; - ops[1] = def->rs2; - ops[2] = def->rs3; - for (int k = 0; k < 3; k++) { - if (!sr_varies(ops[k])) - continue; /* a constant or something the loop never writes */ - if (!sr_collect_chain(func, ops[k], chain, len)) - return false; - } - return true; -} - -/* Add "@var += @step" immediately before @latch's terminator. */ -void sr_emit_advance(basic_block_t *latch, var_t *var, int step) -{ - block_t *scope = latch->scope; - var_t *amount = new_const_var(scope, step); - var_t *sum = require_var(scope); - insn_t *after = latch->insn_list.tail; - insn_t *load; - insn_t *add; - insn_t *assign; - - /* A loop latch ends in the jump back to its header. Appending the advance - * after that jump leaves it unreachable, so place the whole sequence before - * the terminator instead. - */ - if (after && (after->opcode == OP_branch || after->opcode == OP_jump || - after->opcode == OP_return || after->opcode == OP_func_ret)) - after = after->prev; - - sum->var_name = gen_name(); - sum->type = var->type; - sum->ptr_level = var->ptr_level; - - load = new_insn(OP_load_constant, amount, NULL, NULL); - add = new_insn(OP_add, sum, var, amount); - assign = new_insn(OP_assign, var, sum, NULL); - bb_insert_after(latch, after, load); - bb_insert_after(latch, load, add); - bb_insert_after(latch, add, assign); -} - -/* Turn the address @use reads or writes into a pointer the loop advances. - * Reports whether it did. - */ -bool sr_reduce_access(func_t *func, - basic_block_t *pre, - basic_block_t *latch, - insn_t *use) -{ - /* One slot past the chain holds the access itself, so that the scan below - * can be told to ignore the whole of what is moving in one list. - */ - insn_t *chain[MAX_IV_CHAIN + 1]; - int len = 0, step = 0; - var_t *addr = use->rs1; - - if (!addr || !sr_step_of(addr, &step) || !step) - return false; - if (!sr_collect_chain(func, addr, chain, &len) || !len) - return false; - if (len < MIN_IV_CHAIN) - return false; - - /* Every value the chain computes has to belong to it alone: what moves out - * of the loop stops being recomputed there. - * - * The access is skipped along with the chain, so its own operands are - * checked by hand: it may read the address the chain ends at, and nothing - * else. A value the chain computes that the access also stores would stay - * behind in the preheader at its first-iteration value while the address - * went on advancing. - */ - chain[len] = use; - for (int i = 0; i < len; i++) { - var_t *val = chain[i]->rd; - - if (use->rs2 == val || use->rs3 == val) - return false; - if (use->rs1 == val && val != addr) - return false; - if (var_read_by(func, val, chain, len + 1, false)) - return false; - } - - /* Order the chain so that each value lands after the ones it reads. - * - * The walk collected every definition before its operands, so reversing it - * is nearly right -- but only while the chain is a straight line. A value - * two of them share is collected under the first, and reversing then puts - * it after the second, which would read a variable nothing had defined yet. - * Choosing repeatedly instead is correct either way: take an entry once - * every chain value it reads has been taken. Nothing is moved until the - * whole order is settled, so a chain that cannot be ordered at all is - * declined rather than half-moved. - */ - int order[MAX_IV_CHAIN + 1]; - int placed[MAX_IV_CHAIN + 1]; - int n_placed = 0; - - for (int i = 0; i < len; i++) - placed[i] = 0; - - while (n_placed < len) { - int progress = 0; - - for (int i = len - 1; i >= 0; i--) { - if (placed[i]) - continue; - - var_t *ops[3]; - int ready = 1; - - ops[0] = chain[i]->rs1; - ops[1] = chain[i]->rs2; - ops[2] = chain[i]->rs3; - for (int k = 0; k < 3; k++) { - if (!ops[k]) - continue; - for (int j = 0; j < len; j++) { - if (j != i && !placed[j] && chain[j]->rd == ops[k]) - ready = 0; - } - } - if (!ready) - continue; - - order[n_placed++] = i; - placed[i] = 1; - progress = 1; - } - if (!progress) - return false; - } - - for (int i = 0; i < n_placed; i++) { - insn_t *moving = chain[order[i]]; - - bb_remove_insn(moving->belong_to, moving); - bb_append_insn(pre, moving); - } - - sr_emit_advance(latch, addr, step); - return true; -} - -/* Drop the literals the loop no longer needs. - * - * The shift counts and element widths a subscript used are materialised inside - * the loop, and moving the arithmetic that read them out of it leaves those - * materialisations behind with nothing to serve -- four of the fourteen - * instructions in matmul's innermost loop. - */ -void sr_sweep_dead_consts(func_t *func) -{ - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - if (bb->loop_mark != sr_gen) - continue; - - insn_t *next; - for (insn_t *insn = bb->insn_list.head; insn; insn = next) { - next = insn->next; - if (insn->opcode != OP_load_constant || !insn->rd) - continue; - if (var_in_memory(insn->rd)) - continue; - if (var_read_by(func, insn->rd, NULL, 0, true)) - continue; - bb_remove_insn(bb, insn); - } - } -} - -/* The one block outside the loop that falls into @header, or NULL. */ -basic_block_t *sr_preheader(basic_block_t *header) -{ - basic_block_t *pre = NULL; - - for (int i = 0; i < header->prev_idx; i++) { - basic_block_t *p = header->prev[i].bb; - - if (!p) - continue; - if (p->loop_mark == sr_gen) - continue; /* the latch, or anything else inside */ - if (p->then_ || p->else_ || p->next != header) - return NULL; - if (pre) - return NULL; - pre = p; - } - return pre; -} - -/* How many blocks inside the loop jump back to @header. */ -int sr_latch_count(basic_block_t *header) -{ - int n = 0; - - for (int i = 0; i < header->prev_idx; i++) { - const basic_block_t *p = header->prev[i].bb; - - if (p && p->loop_mark == sr_gen) - n++; - } - return n; -} - -void sr_loop(func_t *func, basic_block_t *header, basic_block_t *latch) -{ - int step = 0, made = 0; - - /* One way back, and one way in that runs everything before the loop: the - * first address is computed there and advanced there. The test on the latch - * costs nothing and comes before the walk that marks the loop. - */ - if (latch->then_ || latch->else_ || latch->next != header) - return; - - sr_gen = loop_scan_gen + 1; - if (!mark_natural_loop(header, latch)) - return; /* the walk ran out of room, so loop_mark says nothing */ - - if (sr_latch_count(header) != 1) - return; - - basic_block_t *pre = sr_preheader(header); - - if (!pre) - return; - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - if (bb->loop_mark != sr_gen) - continue; - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - var_t *base = sr_base(insn->rd); - - if (base) - base->loop_stamp = sr_gen; - } - } - - var_t *iv = sr_basic_iv(func, latch, &step); - - if (!iv || !step) - return; - sr_iv = iv; - sr_set_step(iv, step); - sr_derive_steps(func); - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - if (bb->loop_mark != sr_gen) - continue; - - insn_t *next; - for (insn_t *insn = bb->insn_list.head; insn; insn = next) { - next = insn->next; - if (insn->opcode != OP_read && insn->opcode != OP_write) - continue; - - /* A reduction takes instructions out of this block, so the walk - * starts again rather than following a pointer into the block they - * moved to. - */ - if (made >= MAX_IV_PER_LOOP) - break; - if (sr_reduce_access(func, pre, latch, insn)) { - made++; - next = bb->insn_list.head; - } - } - } - - if (made) - sr_sweep_dead_consts(func); -} - -/* Whether @var is an integer scalar narrower than a register pair. */ -bool ssa_is_narrow_scalar(const var_t *var) -{ - return var && !var->ptr_level && !var->is_func && !var->array_size && - var->type && var->type->size < 8 && !var->type->ptr_level && - !is_record_type(var->type); -} - -/* The int type of @var's signedness, which a narrow operand converts to. */ -type_t *int_type_of(const var_t *var) -{ - return var->type->is_unsigned ? TY_uint : TY_int; -} - -/* Insert before @at a copy of the narrow @var widened to @type, sign- or - * zero-extended by @var's own signedness as the usual arithmetic conversions - * require, and return it. - */ -var_t *widen_before(basic_block_t *bb, insn_t *at, var_t *var, type_t *type) -{ - var_t *wide = require_var(bb->scope); - insn_t *ext; - - wide->var_name = gen_name(); - wide->type = type; - ext = new_insn(OP_sign_ext, wide, var, NULL); - ext->sz = (var->type->size << 16) | 8; - bb_insert_after(bb, at->prev, ext); - return wide; -} - -/* Make @insn define a new variable of @type and convert that into its original - * destination straight after, with @conv (an extension or a truncation). - */ -void convert_result_after(basic_block_t *bb, - insn_t *insn, - type_t *type, - opcode_t conv) -{ - var_t *result = insn->rd; - var_t *inner = require_var(bb->scope); - insn_t *copy; - - inner->var_name = gen_name(); - inner->type = type; - insn->rd = inner; - copy = new_insn(conv, result, inner, NULL); - copy->sz = conv == OP_trunc ? result->type->size : (4 << 16) | 8; - bb_insert_after(bb, insn, copy); -} - -/* Give every register-pair operation operands of its own width. - * - * The 32-bit backends lower a wide operation only when all of its values are - * pairs; with a narrow operand they fall back to the one-word form, which keeps - * no high word. The parser does not convert every operand it builds, and an - * LP64 target never noticed, since its registers are already wide: a long long - * compared with an int temporary, or stored from one, lost the high word on Arm - * and RISC-V. Convert those operands here, after the optimizer, so that the - * allocator and the backends see one width per operation. Where the low word of - * a narrow result is exact -- addition, subtraction, multiplication, bitwise - * operations and left shifts -- a narrow destination is left as it is. - */ -void widen_pair_operands(void) -{ - if (PTR_SIZE >= 8) - return; - - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - if (!func->bbs) - continue; - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - var_t *rd = insn->rd; - bool wide1 = var_needs_register_pair(insn->rs1); - bool wide2 = var_needs_register_pair(insn->rs2); - - switch (insn->opcode) { - case OP_eq: - case OP_neq: - case OP_lt: - case OP_leq: - case OP_gt: - case OP_geq: - if (wide1 && ssa_is_narrow_scalar(insn->rs2)) - insn->rs2 = - widen_before(bb, insn, insn->rs2, insn->rs1->type); - else if (wide2 && ssa_is_narrow_scalar(insn->rs1)) - insn->rs1 = - widen_before(bb, insn, insn->rs1, insn->rs2->type); - break; - case OP_add: - case OP_sub: - case OP_mul: - case OP_div: - case OP_mod: - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: - if (var_needs_register_pair(rd)) { - if (!wide1 && !wide2) { - /* Both operands narrow: the operation is an int one - * whose result is then converted. - */ - if (ssa_is_narrow_scalar(insn->rs1) && - ssa_is_narrow_scalar(insn->rs2)) - convert_result_after( - bb, insn, - insn->rs1->type->is_unsigned || - insn->rs2->type->is_unsigned - ? TY_uint - : TY_int, - OP_sign_ext); - break; - } - if (!wide1 && ssa_is_narrow_scalar(insn->rs1)) - insn->rs1 = - widen_before(bb, insn, insn->rs1, rd->type); - if (!wide2 && ssa_is_narrow_scalar(insn->rs2)) - insn->rs2 = - widen_before(bb, insn, insn->rs2, rd->type); - break; - } - - /* A quotient's low word depends on the high words. */ - if ((insn->opcode == OP_div || insn->opcode == OP_mod) && - ssa_is_narrow_scalar(rd) && (wide1 || wide2)) { - type_t *type = - wide1 ? insn->rs1->type : insn->rs2->type; - - if (!wide1 && ssa_is_narrow_scalar(insn->rs1)) - insn->rs1 = widen_before(bb, insn, insn->rs1, type); - if (!wide2 && ssa_is_narrow_scalar(insn->rs2)) - insn->rs2 = widen_before(bb, insn, insn->rs2, type); - convert_result_after(bb, insn, type, OP_trunc); - } - break; - case OP_lshift: - case OP_rshift: - /* The result has the left operand's type. */ - if (var_needs_register_pair(rd) && - ssa_is_narrow_scalar(insn->rs1)) - convert_result_after(bb, insn, int_type_of(insn->rs1), - OP_sign_ext); - else if (insn->opcode == OP_rshift && wide1 && - ssa_is_narrow_scalar(rd)) - convert_result_after(bb, insn, insn->rs1->type, - OP_trunc); - break; - case OP_negate: - case OP_bit_not: - if (var_needs_register_pair(rd) && - ssa_is_narrow_scalar(insn->rs1)) - convert_result_after(bb, insn, int_type_of(insn->rs1), - OP_sign_ext); - break; - case OP_assign: - case OP_unwound_phi: - if (var_needs_register_pair(rd) && - ssa_is_narrow_scalar(insn->rs1)) - insn->rs1 = widen_before(bb, insn, insn->rs1, rd->type); - break; - case OP_read: - if (var_needs_register_pair(rd) && insn->sz == 4) - convert_result_after(bb, insn, int_type_of(rd), - OP_sign_ext); - break; - case OP_write: - if (insn->sz == 8 && ssa_is_narrow_scalar(insn->rs2)) - insn->rs2 = widen_before(bb, insn, insn->rs2, - insn->rs2->type->is_unsigned - ? TY_ulong_long - : TY_long_long); - break; - case OP_return: - if (var_needs_register_pair(&func->return_def) && - ssa_is_narrow_scalar(insn->rs1)) - insn->rs1 = widen_before(bb, insn, insn->rs1, - func->return_def.type); - break; - default: - break; - } - } - } - } -} - -void strength_reduce(void) -{ - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - if (!func->bbs) - continue; - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - var_t *base = sr_base(insn->rd); - - if (base) - base->def_cnt = 0; - } - } - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - var_t *base = sr_base(insn->rd); - - if (base) - base->def_cnt++; - } - } - - for (basic_block_t *latch = func->bbs; latch; latch = latch->rpo_next) { - basic_block_t *succ[3]; - - succ[0] = latch->next; - succ[1] = latch->then_; - succ[2] = latch->else_; - - for (int k = 0; k < 3; k++) { - if (!succ[k] || succ[k] == latch) - continue; - if (!is_dominate(succ[k], latch)) - continue; - sr_loop(func, succ[k], latch); - } - } - } -} - -void if_convert(void) -{ - /* The guard wraps the body rather than returning early: shecc parses its - * own source, and a target without conditional moves would be left with a - * loop after a return, which it rejects as unreachable code. - */ -#ifdef HAVE_COND_MOVE - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - if (!func->bbs) - continue; - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) - if_convert_bb(func, bb); - } -#endif -} - -void unwind_phi(void) -{ - bb_traversal_args_t *args = arena_alloc_traversal_args(); - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - args->func = func; - args->bb = func->bbs; - - func->visited++; - args->preorder_cb = bb_unwind_phi; - bb_forward_traversal(args); - } -} - -/* Whether @pred is a proper ancestor of @succ in the dominator tree. - * - * The walk goes up from @succ rather than down through everything @pred - * dominates: the answer lies on the one path to the root, where the downward - * search visited @pred's whole subtree and did not stop when it found it. - * dom_prev is the inverse of the dom_next the search followed, so the two agree - * on every pair. - */ -bool is_dominate(const basic_block_t *pred, basic_block_t *succ) -{ - for (basic_block_t *bb = succ; bb; bb = bb->dom_prev) { - if (bb->dom_prev == pred) - return true; - } - return false; -} - -/* For any variable, the basic block that defines it must dominate all the basic - * blocks where it is used; otherwise, it is an invalid cross-block - * initialization. - */ -void bb_check_var_cross_init(func_t *func, basic_block_t *bb) -{ - UNUSED(func); - - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->opcode != OP_allocat) - continue; - - const var_t *var = insn->rd; - ref_block_t *ref; - for (ref = var->ref_block_list.head; ref; ref = ref->next) { - if (ref->bb == bb) - continue; - - if (!is_dominate(bb, ref->bb)) - printf("Warning: Variable '%s' cross-initialized\n", - var->var_name); - } - } -} - -/** - * A variable's initialization lives in a basic block that does not dominate all - * of its uses, so control flow can reach a use without first passing through - * its initialization (i.e., a possibly-uninitialized use). - * - * For Example: // Jumps directly to 'label', skipping the declaration below - * goto label; if (1) { - * // This line is never executed when 'goto' is taken - * int x; - * label: - * // Uses 'x' after its declaration was bypassed - * x = 5; - * } - */ -void check_var_cross_init(void) -{ - bb_traversal_args_t *args = arena_alloc_traversal_args(); - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - args->func = func; - args->bb = func->bbs; - - func->visited++; - args->postorder_cb = bb_check_var_cross_init; - bb_forward_traversal(args); - } -} - -void bb_dump_connection(FILE *fd, - basic_block_t *curr, - basic_block_t *next, - bb_connection_type_t type) -{ - const char *str; - - switch (type) { - case NEXT: - str = "%s_%p:s->%s_%p:n\n"; - break; - case THEN: - str = "%s_%p:sw->%s_%p:n\n"; - break; - case ELSE: - str = "%s_%p:se->%s_%p:n\n"; - break; - default: - fatal("Unknown basic block connection type"); - } - - const char *pred; - const void *pred_id; - if (curr->insn_list.tail) { - pred = "insn"; - pred_id = curr->insn_list.tail; - } else { - pred = "pseudo"; - pred_id = curr; - } - - const char *succ; - const void *succ_id; - if (next->insn_list.tail) { - succ = "insn"; - succ_id = next->insn_list.head; - } else { - succ = "pseudo"; - succ_id = next; - } - - fprintf(fd, str, pred, pred_id, succ, succ_id); -} - -/* escape character for the tag in dot file */ -char *get_insn_op(const insn_t *insn) -{ - switch (insn->opcode) { - case OP_add: - return "+"; - case OP_sub: - return "-"; - case OP_mul: - return "*"; - case OP_div: - return "/"; - case OP_mod: - return "%%"; - case OP_lshift: - return "<<"; - case OP_rshift: - return ">>"; - case OP_eq: - return "=="; - case OP_neq: - return "!="; - case OP_gt: - return ">"; - case OP_lt: - return "<"; - case OP_geq: - return ">="; - case OP_leq: - return "<="; - case OP_bit_and: - return "&"; - case OP_bit_or: - return "|"; - case OP_bit_xor: - return "^"; - case OP_log_and: - return "&&"; - case OP_log_or: - return "||"; - default: - fatal("Unknown opcode in operator string conversion"); - return ""; /* unreachable, but silences compiler warning */ - } -} - -void bb_dump(FILE *fd, func_t *func, basic_block_t *bb) -{ - bb->visited++; - - bool next_ = false, then_ = false, else_ = false; - if (bb->next) - next_ = true; - if (bb->then_) - then_ = true; - if (bb->else_) - else_ = true; - if (then_ && !else_) - printf("Warning: missing false branch\n"); - if (!then_ && else_) - printf("Warning: missing true branch\n"); - if (next_ && (then_ || else_)) - printf("Warning: normal BB with condition\n"); - - fprintf(fd, "subgraph cluster_%p {\n", (void *) bb); - fprintf(fd, "label=\"BasicBlock %p (%s)\"\n", (void *) bb, - bb->bb_label_name); - - insn_t *insn = bb->insn_list.head; - if (!insn) - fprintf(fd, "pseudo_%p [label=\"pseudo\"]\n", (void *) bb); - if (!insn && (then_ || else_)) - printf("Warning: pseudo node should only have NEXT\n"); - - for (; insn; insn = insn->next) { - if (insn->opcode == OP_phi) { - fprintf(fd, "insn_%p [label=", (void *) insn); - fprintf(fd, "<%s%d := PHI(%s%d", - insn->rd->var_name, insn->rd->subscript, - insn->phi_ops->var->var_name, - insn->phi_ops->var->subscript); - - for (phi_operand_t *op = insn->phi_ops->next; op; op = op->next) { - fprintf(fd, ", %s%d", op->var->var_name, - op->var->subscript); - } - fprintf(fd, ")>]\n"); - } else { - char str[DUMP_INSN_LEN]; - switch (insn->opcode) { - case OP_allocat: - sprintf(str, "<%s%d := ALLOC>", insn->rd->var_name, - insn->rd->subscript); - break; - case OP_load_constant: - sprintf(str, "<%s%d := CONST %d>", - insn->rd->var_name, insn->rd->subscript, - insn->rd->init_val); - break; - case OP_load_data_address: - sprintf(str, "<%s%d := [.data] + %d>", - insn->rd->var_name, insn->rd->subscript, - insn->rd->init_val); - break; - case OP_load_rodata_address: - sprintf(str, "<%s%d := [.rodata] + %d>", - insn->rd->var_name, insn->rd->subscript, - insn->rd->init_val); - break; - case OP_address_of: - sprintf(str, "<%s%d := &%s%d>", - insn->rd->var_name, insn->rd->subscript, - insn->rs1->var_name, insn->rs1->subscript); - break; - case OP_assign: - sprintf(str, "<%s%d := %s%d>", - insn->rd->var_name, insn->rd->subscript, - insn->rs1->var_name, insn->rs1->subscript); - break; - case OP_read: - sprintf(str, "<%s%d := (%s%d)>", - insn->rd->var_name, insn->rd->subscript, - insn->rs1->var_name, insn->rs1->subscript); - break; - case OP_write: - if (insn->rs2->is_func) - sprintf(str, "<(%s%d) := %s>", - insn->rs1->var_name, insn->rs1->subscript, - insn->rs2->var_name); - else - sprintf(str, "<(%s%d) := %s%d>", - insn->rs1->var_name, insn->rs1->subscript, - insn->rs2->var_name, insn->rs2->subscript); - break; - case OP_branch: - sprintf(str, "%d>", insn->rs1->var_name, - insn->rs1->subscript); - break; - case OP_jump: - sprintf(str, ""); - break; - case OP_label: - sprintf(str, "