Compiling Expressions 17

This commit is contained in:
nazrin 2025-06-04 19:49:53 +00:00
parent 8fb449825d
commit 41404633da
18 changed files with 546 additions and 64 deletions

12
dub.sdl
View file

@ -4,17 +4,19 @@ authors "tanya"
copyright "Copyright © 2025, tanya"
license "MPL-2.0"
dependency "commandr" version="~>1.1.0"
dependency "colored" version="~>0.0.33"
targetType "executable"
sourcePaths
configuration "clox" {
debugVersions "traceExec"
buildRequirements "requireBoundsCheck" "requireContracts"
/* debugVersions "traceExec" */
debugVersions "printCode"
targetType "executable"
sourcePaths "src/clox" "src/common"
buildRequirements "requireBoundsCheck" "requireContracts"
}
configuration "jlox" {
targetType "executable"
sourcePaths "src/jlox" "src/common"
versions "LoxConcatNonStrings" "LoxExtraNativeFuncs" "LoxPrintMultiple"
buildRequirements "requireBoundsCheck" "requireContracts"
sourcePaths "src/jlox" "src/common"
}

View file

@ -1,7 +1,9 @@
{
"fileVersion": 1,
"versions": {
"colored": "0.0.33",
"commandr": "1.1.0",
"taggedalgebraic": "0.11.23"
"taggedalgebraic": "0.11.23",
"unit-threaded": "2.2.3"
}
}

View file

@ -2,8 +2,11 @@ module clox.chunk;
import std.container.array;
import std.stdio;
import std.algorithm.searching;
import clox.value;
import clox.container.rle;
import clox.container.int24;
enum OpCode : ubyte{
Constant,
@ -14,15 +17,23 @@ enum OpCode : ubyte{
struct Chunk{
Array!ubyte code;
Array!uint lines;
Rle!(Uint24, ubyte) lines;
Array!Value constants;
ubyte addConstant(in Value value) @nogc nothrow {
uint addConstant(in Value value) @nogc nothrow {
long index = constants[].countUntil(value);
if(index >= 0)
return cast(uint)index;
constants ~= value;
return cast(ubyte)((constants.length) - 1);
return cast(uint)((constants.length) - 1);
}
void write(ubyte b, uint line = 0) @nogc nothrow {
ubyte[1] data = [ b ];
write(data, line);
}
void write(ubyte[] b, uint line = 0) @nogc nothrow {
code ~= b;
lines ~= line;
foreach(i; 0 .. b.length)
lines ~= Uint24(line); // TODO could be done without a loop
}
}

View file

@ -2,8 +2,29 @@ module clox.compiler;
import std.stdio;
import clox.scanner;
import clox.scanner, clox.parser, clox.emitter;
import clox.chunk;
import clox.value;
import clox.util;
import clox.parserules;
import clox.dbg;
void compile(string source) @nogc nothrow {
struct Compiler{
Scanner scanner;
Parser parser;
Emitter emitter;
bool compile(string source, Chunk* chunk){
scanner = Scanner(source);
parser = Parser(&this);
emitter = Emitter(&this, chunk);
parser.advance();
parser.expression();
parser.consume(Token.Type.EOF, "Expect end of expression.");
debug writeln(*chunk);
emitter.endCompiler();
return !parser.hadError;
}
}

View file

@ -0,0 +1,26 @@
module clox.container.int24;
struct Uint24{
nothrow: @nogc: @safe:
ubyte[3] data;
static Uint24 opCall(uint n){
import std.bitmanip : nativeToLittleEndian;
Uint24 u3;
assert(n <= 16_777_215);
ubyte[uint.sizeof] d = nativeToLittleEndian!uint(n);
u3.data[0 .. 3] = d[0 .. 3];
return u3;
}
}
uint toUint(Uint24 u3) @nogc nothrow @safe {
import std.bitmanip : littleEndianToNative;
ubyte[4] temp;
temp[0 .. 3] = u3.data;
return littleEndianToNative!uint(temp);
}
unittest{
static assert(Uint24.sizeof == 3);
assert(Uint24(5).toUint == 5);
assert(Uint24(16_777_215).toUint == 16_777_215);
}

62
src/clox/container/rle.d Normal file
View file

@ -0,0 +1,62 @@
module clox.container.rle;
import std.traits;
import std.container.array;
struct Rle(T, L = ubyte) if(isUnsigned!L){
nothrow: @nogc:
align(1) struct Count{
T item;
L num;
}
size_t total;
Array!Count data;
private void pushNew(T item){
data ~= Count(item, 0);
}
size_t push(T item){
if(data.length){
Count* d = &data[(data.length)-1];
if(d.item == item && d.num < L.max){
d.num++;
return total++;
}
}
pushNew(item);
return total++;
}
T opOpAssign(string op: "~")(T rhs){
push(rhs);
return rhs;
}
T opIndex(size_t n) const @safe{
assert(n < total);
size_t c;
for(size_t i; i < n;){
i += data[c].num + 1;
if(i <= n)
c++;
}
return data[c].item;
}
}
unittest{
import clox.container.int24;
auto rl = Rle!(Uint24, ubyte)();
static assert(rl.Count.sizeof == 4);
foreach(i; 0..300){
size_t index = rl.push(Uint24(5));
assert(rl[index].toUint == 5);
}
assert(rl[299].toUint == 5);
foreach(i; 0..30){
size_t index = rl.push(Uint24(0));
assert(rl[index].toUint == 0);
}
assert(rl[0].toUint == 5);
foreach(i; 0..300){
size_t index = rl.push(Uint24(16_777_215));
assert(rl[index].toUint == 16_777_215);
}
}

View file

@ -0,0 +1,26 @@
module clox.container.stack;
struct Stack(T, size_t N){
@nogc: nothrow:
T* top;
T[N] data;
invariant{ assert(top <= data.ptr + N); assert(top >= data.ptr); }
this(int _) @safe{
top = data.ptr;
}
void push(T value){
assert(top < data.ptr + N);
debug assert(*top is T.init);
*(top++) = value;
}
T pop(){
assert(top > data.ptr);
T t = *(--top);
debug *(top) = T.init;
return t;
}
const(T)[] live() const @safe{
return data[0 .. (top - data.ptr)];
}
}

View file

@ -0,0 +1,73 @@
module clox.container.varint;
struct VarUint{
import std.bitmanip;
nothrow: @nogc:
uint i;
ubyte len;
ubyte[4] data;
this(long l) @safe {
if(l < 0b1000_0000){
len = 1;
data[0] = (cast(ubyte)l);
return;
}
if(l < 0b0100_0000__0000_0000){
len = 2;
data[0 .. 2] = nativeToBigEndian(cast(ushort)l);
data[0] |= 0b1000_0000;
return;
}
if(l < 0b0010_0000__0000_0000__0000_0000__0000_0000){
len = 4;
data[0 .. 4] = nativeToBigEndian(cast(uint)l);
data[0] |= 0b1100_0000;
return;
}
assert(0);
}
static VarUint read(const(ubyte)[] data) @safe {
VarUint v;
ubyte a = data[0];
if((data[0] & 0b1000_0000) == 0){
v.i = a;
v.len = 1;
return v;
}
if((a & 0b0100_0000) == 0){
ubyte[2] d = data[0 .. 2];
d[0] &= 0b0111_1111;
v.i = bigEndianToNative!ushort(d);
v.len = 2;
return v;
}
if((a & 0b0010_0000) == 0){
ubyte[4] d = data[0 .. 4];
d[0] &= 0b0011_1111;
v.i = bigEndianToNative!uint(d);
v.len = 4;
return v;
}
assert(0);
}
ubyte[] bytes() @nogc nothrow {
return data[0 .. len];
}
}
unittest{
import std.range;
assert(VarUint(5).bytes.length == 1);
assert(VarUint(127).bytes.length == 1);
assert(VarUint(128).bytes.length == 2);
assert(VarUint(536_870_911).bytes.length == 4);
foreach(ulong i; [
0, 1, 2, 5,
150, 127, 128,
536_870_911,
ushort.max * 100
]){
auto vi = VarUint(i);
assert(i == VarUint.read(vi.bytes).i);
}
}

View file

@ -3,33 +3,41 @@ module clox.dbg;
import std.stdio;
import std.conv;
import std.uni;
import std.format;
import colored;
import clox.chunk;
import clox.value;
import clox.util;
import clox.container.varint;
import clox.container.int24;
debug private ulong simpleInstruction(string name, ulong offset) @nogc nothrow{
debug writeln(name);
private ulong simpleInstruction(string name, ulong offset){
writeln(name.lightCyan);
return offset + 1;
}
debug private ulong constantInstruction(string name, Chunk* chunk, ulong offset) @nogc nothrow{
ubyte constant = chunk.code[offset + 1];
debug writef("%-16s %4d '", name, constant);
debug printValue(chunk.constants[constant]);
debug writeln("'");
return offset + 2;
private ulong constantInstruction(string name, Chunk* chunk, ulong offset){
/* ubyte constant = chunk.code[offset + 1]; */
VarUint constant = VarUint.read(chunk.code.data[offset + 1 .. $]);
/* writeln(constant); */
write("%-16s".format(name).cyan, " %4d ".format(constant.i).lightMagenta, "'");
printValue(chunk.constants[constant.i]);
writeln("'");
return offset + 1 + constant.len;
}
debug void disassembleChunk(Chunk* chunk, string name = "chunk") @nogc nothrow{
debug writefln("== %s ==", name);
void disassembleChunk(Chunk* chunk, string name = "chunk"){
writefln("== %s ==", name);
for(ulong offset = 0; offset < chunk.code.length;)
offset = disassembleInstruction(chunk, offset);
}
debug ulong disassembleInstruction(Chunk* chunk, const ulong offset) @nogc nothrow{
debug writef(" %04d ", offset);
ulong disassembleInstruction(Chunk* chunk, const ulong offset){
write(" %04d ".format(offset).lightGray);
if(offset > 0 && chunk.lines[offset] == chunk.lines[offset - 1]){
debug write(" | ");
write(" | ".darkGray);
} else {
debug writef(" %4d ", chunk.lines[offset]);
write(" %4d ".format(chunk.lines[offset].toUint).lightGray);
}
ubyte instruction = chunk.code[offset];
with(OpCode) switch(instruction){
@ -41,7 +49,7 @@ debug ulong disassembleInstruction(Chunk* chunk, const ulong offset) @nogc nothr
return simpleInstruction(name, offset);
}
default:
debug writefln("Unknown opcode %d", instruction);
writefln("Unknown opcode %d", instruction);
return offset + 1;
}
}

51
src/clox/emitter.d Normal file
View file

@ -0,0 +1,51 @@
module clox.emitter;
import clox.compiler;
import clox.chunk;
import clox.value;
import clox.util;
import clox.dbg;
import clox.container.varint;
struct Emitter{
Compiler* compiler;
Chunk* chunk;
private uint line;
Chunk* currentChunk(){
return chunk;
}
void emit(Args...)(Args args){
static foreach(v; args){{
static if(is(typeof(v) == OpCode)){
auto bytes = v;
} else static if(is(typeof(v) == uint)){
auto bytes = VarUint(v).bytes;
} else {
static assert(0);
}
currentChunk.write(bytes, line);
}}
}
void emitConstant(Value value){
emit(OpCode.Constant, makeConstant(value));
}
void emitReturn(){
emit(OpCode.Return);
}
void endCompiler(){
emitReturn();
debug(printCode){
if(!compiler.parser.hadError)
disassembleChunk(currentChunk());
}
}
uint makeConstant(Value value){
uint constant = chunk.addConstant(value);
return constant;
}
void setLine(uint l){
this.line = l;
}
}

View file

@ -25,7 +25,7 @@ struct Lox{
}
int runPrompt(){
while(true){
write("> ");
write("lox> ");
string line = stdin.readln();
if(!line){
writeln();

66
src/clox/parser.d Normal file
View file

@ -0,0 +1,66 @@
module clox.parser;
import clox.compiler;
import clox.value;
import clox.scanner;
import clox.parserules;
struct Parser{
Compiler* compiler;
Token current, previous;
bool hadError, panicMode;
void errorAtCurrent(string message){
errorAt(current, message);
}
void error(string message){
errorAt(previous, message);
}
void errorAt(in ref Token token, string message){
import core.stdc.stdio;
if(panicMode)
return;
panicMode = true;
fprintf(stderr, "[line %d] Error", token.line);
if(token.type == Token.Type.EOF){
fprintf(stderr, " at end");
} else if(token.type != Token.Type.Error){
fprintf(stderr, " at '%.*s'", cast(int)token.lexeme.length, token.lexeme.ptr);
}
fprintf(stderr, ": %.*s\n", cast(int)message.length, message.ptr);
hadError = true;
}
auto consume(Token.Type type, string msg){
if(current.type == type){
advance();
return;
}
errorAtCurrent(msg);
}
void advance(){
previous = current;
while(true){
current = compiler.scanner.scan();
if(current.type != Token.Type.Error)
break;
errorAtCurrent(current.lexeme);
}
}
void expression(){
parsePrecedence(Precedence.Assignment);
}
void parsePrecedence(Precedence precedence){
advance();
ParseFn prefixRule = ParseRule.get(previous.type).prefix;
if(prefixRule == null){
error("Expect expression.");
return;
}
prefixRule(compiler);
while(precedence <= ParseRule.get(current.type).precedence){
advance();
ParseFn infixRule = ParseRule.get(previous.type).infix;
infixRule(compiler);
}
}
}

108
src/clox/parserules.d Normal file
View file

@ -0,0 +1,108 @@
module clox.parserules;
import clox.compiler;
import clox.chunk;
import clox.scanner;
alias ParseFn = void function(Compiler* compiler);
void number(Compiler* compiler){
import core.stdc.stdlib : strtod;
Token token = compiler.parser.previous;
double value = strtod(token.lexeme.ptr, null);
compiler.emitter.setLine(token.line);
compiler.emitter.emitConstant(value);
}
void grouping(Compiler* compiler){
compiler.parser.expression();
compiler.parser.consume(Token.Type.RightParen, "Expect ')' after expression.");
}
void unary(Compiler* compiler){
Token operator = compiler.parser.previous;
compiler.parser.parsePrecedence(Precedence.Unary);
compiler.emitter.setLine(operator.line);
switch(operator.type){
case Token.Type.Minus: compiler.emitter.emit(OpCode.Negate); break;
default: assert(0);
}
}
void binary(Compiler* compiler){
Token operator = compiler.parser.previous;
immutable(ParseRule)* rule = ParseRule.get(operator.type);
compiler.parser.parsePrecedence(cast(Precedence)(rule.precedence + 1));
compiler.emitter.setLine(operator.line);
switch(operator.type){
case Token.Type.Plus: compiler.emitter.emit(OpCode.Add); break;
case Token.Type.Minus: compiler.emitter.emit(OpCode.Subtract); break;
case Token.Type.Star: compiler.emitter.emit(OpCode.Multiply); break;
case Token.Type.Slash: compiler.emitter.emit(OpCode.Divide); break;
default: assert(0);
}
}
struct ParseRule{
ParseFn prefix;
ParseFn infix;
Precedence precedence;
static immutable(ParseRule)* get(Token.Type type) @nogc nothrow{
return &rules[type];
}
}
enum Precedence{
None,
Assignment, // =
Or, // or
And, // and
Equality, // == !=
Comparison, // < > <= >=
Term, // + -
Factor, // * /
Unary, // ! -
Call, // . ()
Primary
}
immutable ParseRule[Token.Type.max+1] rules = [
Token.Type.LeftParen : ParseRule(&grouping, null, Precedence.None),
Token.Type.RightParen : ParseRule(null, null, Precedence.None),
Token.Type.LeftBrace : ParseRule(null, null, Precedence.None),
Token.Type.RightBrace : ParseRule(null, null, Precedence.None),
Token.Type.Comma : ParseRule(null, null, Precedence.None),
Token.Type.Dot : ParseRule(null, null, Precedence.None),
Token.Type.Minus : ParseRule(&unary, &binary, Precedence.Term),
Token.Type.Plus : ParseRule(null, &binary, Precedence.Term),
Token.Type.Semicolon : ParseRule(null, null, Precedence.None),
Token.Type.Slash : ParseRule(null, &binary, Precedence.Factor),
Token.Type.Star : ParseRule(null, &binary, Precedence.Factor),
Token.Type.Bang : ParseRule(null, null, Precedence.None),
Token.Type.BangEqual : ParseRule(null, null, Precedence.None),
Token.Type.Equal : ParseRule(null, null, Precedence.None),
Token.Type.EqualEqual : ParseRule(null, null, Precedence.None),
Token.Type.Greater : ParseRule(null, null, Precedence.None),
Token.Type.GreaterEqual : ParseRule(null, null, Precedence.None),
Token.Type.Less : ParseRule(null, null, Precedence.None),
Token.Type.LessEqual : ParseRule(null, null, Precedence.None),
Token.Type.Identifier : ParseRule(null, null, Precedence.None),
Token.Type.String : ParseRule(null, null, Precedence.None),
Token.Type.Number : ParseRule(&number, null, Precedence.None),
Token.Type.And : ParseRule(null, null, Precedence.None),
Token.Type.Class : ParseRule(null, null, Precedence.None),
Token.Type.Else : ParseRule(null, null, Precedence.None),
Token.Type.False : ParseRule(null, null, Precedence.None),
Token.Type.For : ParseRule(null, null, Precedence.None),
Token.Type.Fun : ParseRule(null, null, Precedence.None),
Token.Type.If : ParseRule(null, null, Precedence.None),
Token.Type.Nil : ParseRule(null, null, Precedence.None),
Token.Type.Or : ParseRule(null, null, Precedence.None),
Token.Type.Print : ParseRule(null, null, Precedence.None),
Token.Type.Return : ParseRule(null, null, Precedence.None),
Token.Type.Super : ParseRule(null, null, Precedence.None),
Token.Type.This : ParseRule(null, null, Precedence.None),
Token.Type.True : ParseRule(null, null, Precedence.None),
Token.Type.Var : ParseRule(null, null, Precedence.None),
Token.Type.While : ParseRule(null, null, Precedence.None),
Token.Type.Error : ParseRule(null, null, Precedence.None),
Token.Type.EOF : ParseRule(null, null, Precedence.None),
];

View file

@ -7,6 +7,7 @@ import common.util;
struct Token{
enum Type : ubyte {
None, Error, EOF, // Special
LeftParen, RightParen, // Single-character tokens.
LeftBrace, RightBrace,
Comma, Dot, Minus, Plus,
@ -20,19 +21,20 @@ struct Token{
For, Fun, If, Nil, Or,
Print, Return, Super, This,
True, Var, While,
Error, EOF // Special
}
Type type;
int line;
string lexeme;
static Token error(string msg) => Token(Token.Type.Error, 0, msg);
static Token error(string msg) nothrow @nogc => Token(Token.Type.Error, 0, msg);
}
struct Scanner{
nothrow:
@nogc:
string start;
string current;
int line = 1;
this(string source){
this(string source) @nogc nothrow{
start = current = source;
}
bool isAtEnd() const => current.length == 0;
@ -42,6 +44,7 @@ struct Scanner{
Token token;
token.type = type;
token.lexeme = start[0 .. current.ptr - start.ptr];
token.line = line;
return token;
}
private char advance(){
@ -67,6 +70,7 @@ struct Scanner{
}
if(!c.isWhite)
return;
/* debug writeln(c == '\n'); */
if(c == '\n')
line++;
current = current[1 .. $];
@ -147,7 +151,7 @@ struct Scanner{
}
if(c.isDigit)
return parseNumber();
return Token.error("Unexpected character '" ~ c ~ "'.");
return Token.error("Unexpected character.");
}
}

View file

@ -1,25 +1,6 @@
module clox.util;
struct Stack(T, size_t N){
T* top;
T[N] data;
invariant{ assert(top <= data.ptr + N); assert(top >= data.ptr); }
this(int _) @nogc nothrow{
top = data.ptr;
}
void push(T value) @nogc nothrow{
assert(top < data.ptr + N);
debug assert(*top is T.init);
*(top++) = value;
}
T pop() @nogc nothrow{
assert(top > data.ptr);
T t = *(--top);
debug *(top) = T.init;
return t;
}
const(T)[] live() @nogc nothrow const{
return data[0 .. (top - data.ptr)];
}
}
import std.stdio;
import std.traits : isUnsigned;
import std.container.array;

View file

@ -4,7 +4,7 @@ import std.stdio;
alias Value = double;
debug void printValue(Value value) @nogc nothrow{
debug writef("%g", value);
void printValue(Value value){
writef("%g", value);
}

View file

@ -7,6 +7,8 @@ import clox.value;
import clox.dbg;
import clox.util;
import clox.compiler;
import clox.container.stack;
import clox.container.varint;
enum stackMax = 256;
@ -18,11 +20,15 @@ struct VM{
this(int _) @nogc nothrow {
stack = typeof(stack)(0);
}
InterpretResult interpret(string source) @nogc nothrow {
compile(source);
return InterpretResult.Ok;
InterpretResult interpret(string source){
Chunk c = Chunk();
Compiler compiler;
if(!compiler.compile(source, &c))
return InterpretResult.CompileError;
chunk = &c;
return interpret(chunk);
}
InterpretResult interpret(Chunk* chunk) @nogc nothrow {
InterpretResult interpret(Chunk* chunk){
this.chunk = chunk;
ip = &chunk.code[0];
return run();
@ -30,11 +36,15 @@ struct VM{
private InterpretResult run() @nogc nothrow {
auto readByte() => *ip++;
auto readIns() => cast(OpCode)readByte();
auto readConstant() => chunk.constants[readByte()];
Value readConstant(){
VarUint constant = VarUint.read(ip[0 .. 4]);
ip += constant.len;
return chunk.constants[constant.i];
}
while(true){
debug(traceExec){
writeln(" ", stack.live);
debug disassembleInstruction(chunk, ip - &chunk.code[0]);
disassembleInstruction(chunk, ip - &chunk.code[0]);
}
OpCode instruction = readIns();
with(OpCode) opSwitch: final switch(instruction){

View file

@ -8,6 +8,37 @@ template defaultCtor(){
}
import std.ascii : isAlpha, isAlphaNum;
bool isAlpha_(dchar c) => c.isAlpha || c == '_';
bool isAlphaNum_(dchar c) => c.isAlphaNum || c == '_';
bool isAlpha_(dchar c) @nogc nothrow @safe => c.isAlpha || c == '_';
bool isAlphaNum_(dchar c) @nogc nothrow @safe => c.isAlphaNum || c == '_';
T pop(T)(ref T[] arr){
T v = arr.last;
arr.length--;
return v;
}
T shift(T)(ref T[] arr){
T v = arr.first;
arr = arr[1 .. $];
return v;
}
ref T sole(T)(T[] arr) @nogc @safe pure{
assert(arr.length == 1, "Not sole");
return arr[0];
}
ref T first(T)(T[] arr) @nogc @safe pure{
assert(arr.length >= 1, "No first");
return arr[0];
}
ref T second(T)(T[] arr) @nogc @safe pure{
assert(arr.length >= 2, "No second");
return arr[1];
}
ref T third(T)(T[] arr) @nogc @safe pure{
assert(arr.length >= 3, "No third");
return arr[2];
}
ref T last(T)(T[] arr) @nogc @safe pure{
assert(arr.length >= 1, "No last");
return arr[$-1];
}