Strings 19

.gitignore

@@ -17,4 +17,5 @@ lox-test-*
 
 .msc/
 test/test.lox
+.ccls-cache/
 

dub.sdl

@@ -8,7 +8,8 @@ dependency "colored" version="~>0.0.33"
 targetType "executable"
 sourcePaths
 configuration "clox" {
-	/* debugVersions "traceExec" */
+	debugVersions "traceExec"
+	libs "libbacktrace"
 	debugVersions "printCode"
 	targetType "executable"
 	sourcePaths "src/clox" "src/common"

src/clox/chunk.d

@@ -2,9 +2,10 @@ module clox.chunk;
 
 import std.container.array;
 import std.stdio;
-import std.algorithm.searching;
+import std.algorithm;
 
 import clox.value;
+import clox.object;
 import clox.container.rle;
 import clox.container.int24;
 
@@ -49,8 +50,21 @@ struct Chunk{
 	Array!ubyte code;
 	Rle!(Uint24, ubyte) lines;
 	Array!Value constants;
+	string name;
+	~this(){
+		stderr.writeln("Deallocing chunk ", name);
+		foreach(value; constants[].filter!isObj)
+			value.getObj.freeObject();
+	}
 	uint addConstant(in Value value) @nogc nothrow {
-		long index = constants[].countUntil(value);
+		long index;
+		switch(value.type){
+			case Value.Type.Str:
+				index = constants[].countUntil!(v => v.isStr && v.getStr.data == value.getStr.data);
+				break;
+			default:
+				index = constants[].countUntil(value);
+		}
 		if(index >= 0)
 			return cast(uint)index;
 		constants ~= value;
@@ -58,7 +72,7 @@ struct Chunk{
 	}
 	void write(ubyte b, uint line = 0) @nogc nothrow {
 		ubyte[1] data = [ b ];
-		write(data, line);
+		this.write(data, line);
 	}
 	void write(ubyte[] b, uint line = 0) @nogc nothrow {
 		code ~= b;

src/clox/compiler.d

@@ -21,8 +21,8 @@ struct Compiler{
 		parser.advance();
 		parser.expression();
 		parser.consume(Token.Type.EOF, "Expect end of expression.");
-		debug writeln(*chunk);
 		emitter.endCompiler();
+		
 		return !parser.hadError;
 	}
 }

src/clox/container/vartype.d

@@ -17,10 +17,23 @@ struct VarType(S) if(is(S == union)){
 	}
 	private template funcs(string G, string T){
 		mixin("bool is", G, "() const nothrow @nogc @safe => this.type == this.Type.", G, ";");
-		mixin("auto get", G, "() const nothrow @nogc { check(this.Type.", G, "); return value.", T, "; }");
+		mixin("auto get", G, "() const nothrow @nogc {
-		mixin("void set", G, "(typeof(S.", T, ") v = typeof(S.", T, ").init){ this._type = this.Type.", G, "; this.value.", T, " = v; }");
+			check(this.Type.", G, ");
-		mixin("static auto ", T, "(typeof(S.", T, ") v){ typeof(this) vt; vt.set", G, "(v); return vt; }");
+			return value.", T, ";
-		mixin("static auto ", T, "(){ typeof(this) vt; vt.set", G, "(); return vt; }");
+		}");
+		mixin("void set", G, "(typeof(S.", T, ") v = typeof(S.", T, ").init){
+			this._type = this.Type.", G, ";
+			this.value.", T, " = v;
+		}");
+		mixin("static auto ", T, "(typeof(S.", T, ") v){
+			typeof(this) vt; vt.set", G, "(v);
+			return vt;
+		}");
+		mixin("static auto ", T, "(){
+			typeof(this) vt;
+			vt.set", G, "();
+			return vt;
+		}");
 	}
 	static foreach(s; members){
 		mixin funcs!(s.asCapitalized.to!string, s);
@@ -28,7 +41,8 @@ struct VarType(S) if(is(S == union)){
 	string toString() const{
 		final switch(_type){
 			static foreach(s; members){
-				mixin("case Type.", s.asCapitalized.to!string, ": return _type.to!string ~ ':' ~ get", s.asCapitalized.to!string, ".to!string ;");
+				mixin("case Type.", s.asCapitalized.to!string, ":
+					return _type.to!string ~ ':' ~ get", s.asCapitalized.to!string, ".to!string ;");
 			}
 			case Type.None: return "None";
 		}

src/clox/dbg.d

@@ -4,6 +4,8 @@ import std.stdio;
 import std.conv;
 import std.uni;
 import std.format;
+import std.meta : Filter;
+import std.traits : EnumMembers;
 
 import colored;
 
@@ -15,42 +17,39 @@ import clox.container.int24;
 
 private ulong simpleInstruction(alias op)(string name, ulong offset){
 	static if(isValueOp!op)
-		writeln(name.cyan);
+		stderr.writeln(name.cyan);
 	else static if(isLogicOp!op)
-		writeln(name.lightRed);
+		stderr.writeln(name.lightRed);
 	else static if(isCompOp!op)
-		writeln(name.red);
+		stderr.writeln(name.red);
 	else static if(isArithOp!op)
-		writeln(name.yellow);
+		stderr.writeln(name.yellow);
 	else
-		writeln(name.lightCyan);
+		stderr.writeln(name.lightCyan);
 	return offset + 1;
 }
 private ulong constantInstruction(string name, Chunk* chunk, ulong offset){
-	/* ubyte constant = chunk.code[offset + 1]; */
 	VarUint constant = VarUint.read(chunk.code.data[offset + 1 .. $]);
-	/* writeln(constant); */
+	stderr.write("%-16s".format(name).cyan, " %4d ".format(constant.i).lightMagenta, "'");
-	write("%-16s".format(name).cyan, " %4d ".format(constant.i).lightMagenta, "'");
 	printValue(chunk.constants[constant.i]);
-	writeln("'");
+	stderr.writeln("'");
 	return offset + 1 + constant.len;
 }
 
 void disassembleChunk(Chunk* chunk, string name = "chunk"){
-	writefln("== %s ==", name);
+	stderr.writefln("== %s ==", name);
 	for(ulong offset = 0; offset < chunk.code.length;)
 		offset = disassembleInstruction(chunk, offset);
 }
 ulong disassembleInstruction(Chunk* chunk, const ulong offset){
-	write(" %04d ".format(offset).lightGray);
+	stderr.write(" %04d ".format(offset).lightGray);
 	if(offset > 0 && chunk.lines[offset] == chunk.lines[offset - 1]){
-		write("    | ".darkGray);
+		stderr.write("    | ".darkGray);
 	} else {
-		write(" %4d ".format(chunk.lines[offset].toUint).lightGray);
+		stderr.write(" %4d ".format(chunk.lines[offset].toUint).lightGray);
 	}
 	ubyte instruction = chunk.code[offset];
 	with(OpCode) switch(instruction){
-		import std.meta, std.traits;
 		case Constant:
 			return constantInstruction("OP_CONSTANT", chunk, offset);
 		static foreach(k; Filter!(isSize1Op, EnumMembers!OpCode)){
@@ -59,7 +58,7 @@ ulong disassembleInstruction(Chunk* chunk, const ulong offset){
 				return simpleInstruction!k(name, offset);
 		}
 		default:
-			writefln("Unknown opcode %d", instruction);
+			stderr.writefln("Unknown opcode %d", instruction);
 			return offset + 1;
 	}
 }

src/clox/emitter.d

@@ -12,10 +12,10 @@ struct Emitter{
 	Compiler* compiler;
 	Chunk* chunk;
 	private uint line = 1;
-	Chunk* currentChunk(){
+	Chunk* currentChunk() @nogc nothrow {
 		return chunk;
 	}
-	void emit(Args...)(Args args){
+	void emit(Args...)(Args args) @nogc nothrow {
 		static foreach(v; args){{
 			static if(is(typeof(v) == OpCode)){
 				auto bytes = v;
@@ -27,10 +27,10 @@ struct Emitter{
 			currentChunk.write(bytes, line);
 		}}
 	}
-	void emitConstant(Value value){
+	void emitConstant(Value value) @nogc nothrow {
 		emit(OpCode.Constant, makeConstant(value));
 	}
-	void emitReturn(){
+	void emitReturn() @nogc nothrow {
 		emit(OpCode.Return);
 	}
 	void endCompiler(){
@@ -40,11 +40,11 @@ struct Emitter{
 				disassembleChunk(currentChunk());
 		}
 	}
-	uint makeConstant(Value value){
+	uint makeConstant(Value value) @nogc nothrow {
 		uint constant = chunk.addConstant(value);
 		return constant;
 	}
-	void setLine(uint l){
+	void setLine(uint l) @nogc nothrow {
 		this.line = l;
 	}
 }

src/clox/main.d

@@ -6,6 +6,7 @@ import std.file;
 import clox.chunk;
 import clox.dbg;
 import clox.vm;
+import clox.object;
 
 extern(C) int isatty(int);
 

src/clox/mem.d

@@ -0,0 +1,152 @@
+module clox.mem;
+
+import std.stdio;
+import std.traits;
+import std.conv;
+import core.stdc.stdlib : calloc, realloc, free;
+import std.range, std.algorithm;
+
+import colored;
+
+import clox.util;
+
+debug {
+	private:
+	alias backtrace_state = void; // "This struct is intentionally not defined in the public interface"
+	alias uintptr_t = void*;
+	extern(C) alias backtrace_error_callback = extern(C) void function(void *data, const char *msg, int errnum) nothrow;
+	extern(C) backtrace_state* backtrace_create_state(const char *filename, int threaded, backtrace_error_callback error_callback, void *data) nothrow;
+	extern(C) extern void backtrace_print(backtrace_state *state, int skip, FILE *) nothrow;
+	extern(C) alias backtrace_full_callback = extern(C) int function(void *data, uintptr_t pc, const char *filename, int lineno, const char* func) nothrow;
+	extern(C) int backtrace_full(backtrace_state* state, int skip, backtrace_full_callback callback, backtrace_error_callback error_callback, void *data) nothrow;
+
+	backtrace_state* bts;
+
+	extern(C) void btsErrCB(void* data, const char* msg, int errnum) nothrow{
+		debug stderr.writeln("BT: ", msg.to!string);
+	}
+	extern(C) int btsFullCB(void* data, uintptr_t pc, const char* filename, int lineno, const char* func) nothrow{
+		import std.demangle;
+		BackTraceBuilder* btb = cast(BackTraceBuilder*)data;
+		if(func){
+			string funcStr = func.to!string.demangle;
+			btb.funcStack ~= funcStr;
+			btb.lineStack ~= lineno;
+		}
+		return 0;
+	}
+	static this(){
+		bts = backtrace_create_state(null, false, &btsErrCB, null).validateAssert();
+	}
+
+	struct BackTraceBuilder{
+		string[] funcStack;
+		int[] lineStack;
+		string toString() const{
+			return zip(funcStack, lineStack).map!"'\t' ~ a[0] ~ ` : ` ~ a[1].to!string".join("\n");
+		}
+	}
+	BackTraceBuilder createBacktrace(int skip = 1) nothrow{
+		BackTraceBuilder btb;
+		backtrace_full(bts, skip, &btsFullCB, &btsErrCB, &btb).validateAssert!"!a";
+		return btb;
+	}
+	
+	struct Allocation{
+		ulong size;
+		BackTraceBuilder backtrace;
+	}
+	Allocation[void*] allocatedPointers;
+
+	ulong totalAllocs, totalReallocs, totalFrees;
+	ulong totalAllocBytes, totalFreedBytes;
+	static ~this(){
+		stderr.writefln("Allocs: %d (%d bytes), Reallocs: %d, Frees: %d (%d bytes)", totalAllocs, totalAllocBytes, totalReallocs, totalFrees, totalFreedBytes);
+		if(totalAllocBytes > totalFreedBytes){
+			stderr.writefln("Leaked %d bytes!".lightRed.to!string, totalAllocBytes - totalFreedBytes);
+
+			foreach(ptr, alloc; allocatedPointers.byPair.filter!"a[1].size"){
+				stderr.writeln(ptr.to!string.red, " ", alloc.size.to!string.magenta, " bytes");
+				stderr.writeln(alloc.backtrace);
+			}
+		}
+	}
+}
+
+T* allocate(T)() nothrow @nogc{
+	return allocate!T(1).ptr;
+}
+T[] allocate(T)(size_t n) nothrow @nogc{
+	T* data = cast(T*)calloc(n, T.sizeof);
+	assert(data);
+
+	debug {
+		allocatedPointers[data] = Allocation(n * T.sizeof, createBacktrace());
+		totalAllocs++;
+		totalAllocBytes += n * T.sizeof;
+	}
+
+	return data[0 .. n];
+}
+void reallocate(T)(ref T[] arr, size_t newSize) nothrow @nogc{
+	debug {
+		assert(arr.ptr, "Null pointer reallocate");
+		assert(arr.ptr in allocatedPointers, "Invalid ptr");
+	}
+	if(arr.length == newSize)
+		return;
+
+	T* newPtr = cast(T*)arr.ptr.realloc(newSize * T.sizeof);
+	assert(newPtr);
+
+	debug if(arr.ptr != newPtr){
+		totalReallocs++;
+		totalFreedBytes += allocatedPointers[arr.ptr].size;
+		totalAllocBytes += newSize;
+		allocatedPointers[newPtr] = Allocation(newSize);
+		allocatedPointers[arr.ptr] = Allocation(0);
+	}
+
+	if(newSize > arr.length){
+		foreach(i; arr.length .. newSize)
+			newPtr[i] = 0;
+	}
+
+	T[] newArr = newPtr[0 .. newSize];
+	arr = newArr;
+}
+void deallocate(T)(T* ptr) nothrow @nogc {
+	debug {
+		assert(ptr, "Null pointer free");
+		assert(ptr in allocatedPointers, "Invalid ptr");
+		assert(allocatedPointers[ptr].size, "Double free");
+		totalFrees++;
+		totalFreedBytes += allocatedPointers[ptr].size;
+		allocatedPointers[ptr] = Allocation(0);
+	}
+	ptr.free();
+}
+void deallocate(T)(ref T[] arr) nothrow @nogc{
+	arr.ptr.deallocate();
+	arr = [];
+}
+
+unittest{
+	int[] i = allocate!int(1);
+
+	i[0] = 5;
+
+	i.reallocate(64);
+	i.reallocate(2);
+
+	assert(i == [ 5, 0 ]);
+
+	i.deallocate();
+
+	assert(i == []);
+
+	int* ip = allocate!int;
+	assert(ip);
+	ip.deallocate();
+}
+

src/clox/object.d

@@ -0,0 +1,79 @@
+module clox.object;
+
+import std.stdio;
+import std.conv;
+import core.stdc.string : memcpy, memcmp, strcat;
+
+import clox.mem;
+import clox.vm;
+import clox.value;
+
+struct Obj{
+	enum Type{
+		String,
+	}
+	Type type;
+	Obj* next;
+	static T* create(T)(VM* vm = null) @nogc nothrow{
+		T* obj = cast(T*)allocate!T();
+		obj.obj.type = T.type;
+		return obj;
+	}
+	static struct String{
+		static enum type = Type.String;
+		Obj obj;
+		char[] data;
+		static String* create(size_t len, VM* vm = null) @nogc nothrow{
+			String* str = Obj.create!String();
+			if(vm){
+				str.obj.next = vm.objects;
+				vm.objects = &str.obj;
+			}
+			str.data = allocate!char(len + 1);
+			return str;
+		}
+		static String* copy(string s, VM* vm = null) nothrow @nogc{
+			String* str = String.create(s.length, vm);
+			str.data.ptr.memcpy(s.ptr, s.length);
+			return str;
+		}
+		static String* concat(const(String)* a, const(String)* b, VM* vm) nothrow @nogc{
+			String* newStr = String.create((a.data.length)-1 + (b.data.length)-1, vm);
+			newStr.data.ptr.memcpy(a.data.ptr, a.data.length);
+			newStr.data.ptr.strcat(b.data.ptr);
+			return newStr;
+		}
+	}
+	string toString() const{
+		final switch(type){
+			case Type.String:
+				const(String)* str = cast(String*)&this;
+				return cast(immutable)str.data;
+		}
+	}
+}
+
+bool isObj(Value value){
+	switch(value.type){
+		case value.Type.Str: return true;
+		default: return false;
+	}
+}
+Obj* getObj(Value value){
+	switch(value.type){
+		case value.Type.Str: return cast(Obj*)value.getStr;
+		default: assert(0);
+	}
+}
+void freeObject(Obj.String* str) @nogc nothrow{
+	str.data.deallocate();
+	str.deallocate();
+}
+void freeObject(Obj* obj) @nogc nothrow{
+	final switch(obj.type){
+		case Obj.Type.String:
+			freeObject(cast(Obj.String*)obj);
+			break;
+	}
+}
+

src/clox/parserules.d

@@ -4,6 +4,7 @@ import clox.compiler;
 import clox.chunk;
 import clox.scanner;
 import clox.value;
+import clox.object;
 
 alias ParseFn = void function(Compiler* compiler);
 
@@ -52,13 +53,22 @@ private void binary(Compiler* compiler){
 	}
 }
 private void literal(Compiler* compiler){
-	switch(compiler.parser.previous.type){
+	Token token = compiler.parser.previous;
+	compiler.emitter.setLine(token.line);
+	switch(token.type){
 		case Token.Type.True: compiler.emitter.emit(OpCode.True); break;
 		case Token.Type.False: compiler.emitter.emit(OpCode.False); break;
 		case Token.Type.Nil: compiler.emitter.emit(OpCode.Nil); break;
 		default: assert(0);
 	}
 }
+private void strlit(Compiler* compiler) @nogc{
+	Token token = compiler.parser.previous;
+	string str = token.lexeme[1 .. $-1];
+	Obj.String* strObj = Obj.String.copy(str);
+	compiler.emitter.setLine(token.line);
+	compiler.emitter.emitConstant(Value.str(strObj));
+}
 
 struct ParseRule{
 	ParseFn prefix;
@@ -104,7 +114,7 @@ immutable ParseRule[Token.Type.max+1] rules = [
 	Token.Type.Less          : ParseRule(null,      &binary, Precedence.Comparison),
 	Token.Type.LessEqual     : ParseRule(null,      &binary, Precedence.Comparison),
 	Token.Type.Identifier    : ParseRule(null,      null,    Precedence.None),
-	Token.Type.String        : ParseRule(null,      null,    Precedence.None),
+	Token.Type.String        : ParseRule(&strlit,   null,    Precedence.None),
 	Token.Type.Number        : ParseRule(&number,   null,    Precedence.None),
 	Token.Type.And           : ParseRule(null,      null,    Precedence.None),
 	Token.Type.Class         : ParseRule(null,      null,    Precedence.None),

src/clox/scanner.d

@@ -70,14 +70,13 @@ struct Scanner{
 			}
 			if(!c.isWhite)
 				return;
-			/* debug writeln(c == '\n'); */
 			if(c == '\n')
 				line++;
 			current = current[1 .. $];
 		}
 	}
 	private Token parseString(){
-		while(peek() != '"' && !isAtEnd){
+		while(!isAtEnd && peek() != '"'){
 			if(peek() == '\n')
 				line++;
 			advance();

src/clox/util.d

@@ -3,4 +3,18 @@ module clox.util;
 import std.stdio;
 import std.traits : isUnsigned;
 import std.container.array;
+import std.functional : unaryFun;
+
+T validateAssert(alias pred = "!!a", T)(T v, lazy string msg = null) nothrow {
+	try{
+		string m = msg;
+		static if(is(typeof(pred) == string))
+			m = msg ? msg : pred;
+		assert(v.unaryFun!pred, m);
+		return v;
+	} catch(Exception){
+		assert(0);
+	}
+}
+
 

src/clox/value.d

@@ -1,31 +1,27 @@
 module clox.value;
 
 import std.stdio;
+import std.conv;
 
+import colored;
+
+import clox.object;
 import clox.container.vartype;
 
-/* struct Value{ */
-/* 	alias T = VarType!u; */
-/* 	private union U{ */
-/* 		bool bln; */
-/* 		bool nil; */
-/* 		double num; */
-/* 	} */
-/* 	T v; */
-/* 	alias v this; */
-/* } */
 private union U{
 	bool bln;
 	bool nil;
 	double num;
+	Obj.String* str;
 }
 alias Value = VarType!U;
 
 void printValue(Value value){
 	final switch(value.type){
-		case value.Type.Bln: writef("%s", value.getBln); break;
+		case value.Type.Bln: stderr.writef("%s", value.getBln.to!string.yellow); break;
-		case value.Type.Num: writef("%g", value.getNum); break;
+		case value.Type.Num: stderr.writef("%g", value.getNum); break;
-		case value.Type.Nil: writef("nil"); break;
+		case value.Type.Nil: stderr.writef("nil"); break;
+		case value.Type.Str: stderr.writef("%s", value.getStr.data); break;
 		case value.Type.None: assert(0);
 	}
 }
@@ -34,6 +30,7 @@ bool isTruthy(Value value) nothrow @nogc {
 		case value.Type.Bln: return value.getBln;
 		case value.Type.Num: return true;
 		case value.Type.Nil: return false;
+		case value.Type.Str: return true;
 		case value.Type.None: assert(0);
 	}
 }
@@ -46,6 +43,7 @@ bool compare(string op)(Value a, Value b){
 	final switch(a.type){
 		case a.Type.Bln: return mixin("a.getBln", op, "b.getBln");
 		case a.Type.Num: return mixin("a.getNum", op, "b.getNum");
+		case a.Type.Str: return mixin("a.getStr.data", op, "b.getStr.data");
 		case a.Type.Nil: return true;
 		case a.Type.None: assert(0);
 	}

src/clox/vm.d

@@ -7,6 +7,7 @@ import clox.value;
 import clox.dbg;
 import clox.util;
 import clox.compiler;
+import clox.object;
 import clox.container.stack;
 import clox.container.varint;
 import clox.container.int24;
@@ -17,10 +18,14 @@ struct VM{
 	const(ubyte)* ip;
 	Stack!(Value, stackMax) stack;
 	Chunk* chunk;
+	Obj* objects;
 	enum InterpretResult{ Ok, CompileError, RuntimeError }
 	this(int _) @nogc nothrow {
 		stack = typeof(stack)(0);
 	}
+	~this(){
+		freeObjects();
+	}
 	InterpretResult interpret(string source){
 		Chunk c = Chunk();
 		Compiler compiler;
@@ -56,7 +61,7 @@ struct VM{
 		}
 		while(true){
 			debug(traceExec){
-				writeln("          ", stack.live);
+				stderr.writeln("          ", stack.live);
 				disassembleInstruction(chunk, ip - &chunk.code[0]);
 			}
 			OpCode instruction = readIns();
@@ -76,6 +81,15 @@ struct VM{
 					break;
 				static foreach(k, op; [ Add: "+", Subtract: "-", Multiply: "*", Divide: "/" ]){
 					case k:
+						static if(k == Add){
+							if(peek(0).isStr && peek(1).isStr){
+								const(Obj.String)* b = stack.pop().getStr;
+								const(Obj.String)* a = stack.pop().getStr;
+								Obj.String* newStr = Obj.String.concat(a, b, &this);
+								stack.push(Value.str(newStr));
+								break opSwitch;
+							}
+						}
 						if(!peek(0).isNum || !peek(1).isNum){
 							runtimeError("Operands must be numbers.");
 							return InterpretResult.RuntimeError;
@@ -104,11 +118,18 @@ struct VM{
 					break;
 				case Return:
 					debug printValue(stack.pop());
-					debug writeln();
+					debug stderr.writeln();
 					return InterpretResult.Ok;
 			}
 		}
 		assert(0);
 	}
+	void freeObjects(){
+		for(Obj* obj = objects; obj !is null;){
+			Obj* next = obj.next;
+			obj.freeObject();
+			obj = next;
+		}
+	}
 }