Storage of names is now fully internal, the NOSTRINGLITERALS and NOARRAY flags are no longer supported

.gitignore

@@ -1,2 +1,3 @@
 cmake-build-*
+build/**
 .idea/*

include/ink.h

@@ -52,7 +52,7 @@ struct stack_frame {
  * Represents a function within ink, defined in ink, using the homoiconic representation of ink
  */
 struct fn {
-	t">char *name;
+	">struct elem name;
 	struct elem *things;
 	int size;
 };
@@ -63,7 +63,7 @@ struct context;
  * Represents natively defined words within ink
  */
 struct native_fn {
-	t">char *name;
+	">struct elem name;
 	void (*value)(struct context *);
 };
 

lib.c

@@ -52,6 +52,16 @@ struct label {
 
 static void noop(void) {}
 
+
+#ifndef NOSTRINGLITERALS
+static void new_protected_array(struct context* ctx);
+static void new_array(struct context* ctx);
+int array_push_s(struct context* ctx, struct ink_array* ary, struct elem value);
+static void print_array_of_codepoints(struct context* ctx);
+static void collect_array(struct context* ctx, void* array);
+static struct ink_collection_list gc_array(struct context* ctx, void* array);
+#endif
+
 #ifdef NOSTDLIB
 
 static size_t strlen(const char* c) {
@@ -132,9 +142,80 @@ static int atoi(const char* c) {
 
 #endif
 
+static int utf8_consume(const char ** name, struct elem * elem) {
+	elem->type = INK_INTEGER;
+	if (**name == '\0') { return 0; }
+	if (!(**name & 128)) {
+		elem->value = **name;
+		*name += 1;
+		return 1;
+	}
+	if(**name & 128 && !(**name & 64)) {
+		elem->value = **name - 128;
+		*name += 1;
+		elem->value = elem->value << 7;
+		elem->value += **name - 128;
+		*name += 1;
+		return 1;
+	}
+	if(**name & 128 && **name & 64 && !(**name & 32)) {
+		elem->value = **name - 128 - 64;
+		*name += 1;
+		elem->value = elem->value << 7;
+		elem->value += **name - 128;
+		*name += 1;
+		elem->value = elem->value << 7;
+		elem->value += **name - 128;
+		*name += 1;
+		return 1;
+	}
+	if(**name & 128 && **name & 64 && **name & 32 && !(**name & 16)) {
+		elem->value = **name - 128 - 64 - 32;
+		*name += 1;
+		elem->value = elem->value << 7;
+		elem->value += **name - 128;
+		*name += 1;
+		elem->value = elem->value << 7;
+		elem->value += **name - 128;
+		*name += 1;
+		elem->value = elem->value << 7;
+		elem->value += **name - 128;
+		*name += 1;
+		return 1;
+	}
+	if(**name & 128 && **name & 64 && **name & 32 && **name & 16 && !(**name & 8)) {
+		elem->value = **name - 128 - 64 - 32;
+		*name += 1;
+		elem->value = elem->value << 7;
+		elem->value += **name - 128;
+		*name += 1;
+		elem->value = elem->value << 7;
+		elem->value += **name - 128;
+		*name += 1;
+		elem->value = elem->value << 7;
+		elem->value += **name - 128;
+		*name += 1;
+		elem->value = elem->value << 7;
+		elem->value += **name - 128;
+		*name += 1;
+		return 1;
+	}
+	elem->value = 0xFFFD;
+	return 1;
+}
+
 int ink_add_native(struct context* ctx, const char* name, void(*value)(struct context*)) {
-	int len;
-	char* copy;
+	int tid;
+	struct elem e;
+	struct ink_array ary;
+	const char* name_it;
+	struct elem character;
+	tid =  get_type_by_name(ctx, "array");
+	ary.elements = NULL;
+	ary.top = 0;
+	ary.capacity = 0;
+	ary.flags = 0;
+
 	if(ctx->native_words == NULL) {
 		ctx->native_words = ctx->inner_malloc(ctx, sizeof(struct native_fn) * 8);
 		ctx->native_words_top = 0;
@@ -151,22 +232,29 @@ int ink_add_native(struct context* ctx, const char* name, void(*value)(struct co
 			ctx->native_words_capacity = new_count;
 		}
 	}
-	len = strlen(name);
-	copy = ctx->inner_malloc(ctx, len+1);
-	if(copy == NULL) {
-		return -4;
+	name_it = name;
+	while (utf8_consume(&name_it, &character)) {
+		array_push_s(ctx, &ary, character);
 	}
-	memcpy(copy, name, len);
-	copy[len] = 0;
+	e = ink_make_native(ctx, tid, &ary);
 	ctx->native_words[ctx->native_words_top].value = value;
-	ctx->native_words[ctx->native_words_top].name = copy;
+	ctx->native_words[ctx->native_words_top].name = e;
 	ctx->native_words_top++;
 	return 0;
 }
 
-static int ink_add_indigenous(struct context* ctx, const char* name, struct elem* m, size_t count) {
-	int len, i;
-	char* copy;
+static int ink_add_indigenous(struct context* ctx, char* name, struct elem* m, size_t count) {
+	int tid;
+	struct elem e;
+	int i;
+	struct ink_array ary;
+	const char* name_it;
+	struct elem character;
+	tid =  get_type_by_name(ctx, "array");
+	ary.elements = NULL;
+	ary.top = 0;
+	ary.capacity = 0;
+	ary.flags = 0;
 	
 	if(ctx->words == NULL) {
 		ctx->words = ctx->malloc(ctx, sizeof(struct fn) * 8);
@@ -184,8 +272,14 @@ static int ink_add_indigenous(struct context* ctx, const char* name, struct elem
 			ctx->words_capacity = new_count;
 		}
 	}
+	name_it = name;
+	while (utf8_consume(&name_it, &character)) {
+		array_push_s(ctx, &ary, character);
+	}
+	e = ink_make_native(ctx, tid, &ary);
 	for(i = 0; i < ctx->words_top; ++i) {
-		if(strcmp(name, ctx->words[i].name) == 0) {
+		struct ink_array* inner_name = ink_get_value(ctx, ctx->words[i].name);
+		if(ary.top == inner_name->top && memcmp(ary.elements, inner_name->elements, sizeof(struct elem)*ary.top) == 0) {
 			if(ctx->words[i].things != NULL) ctx->free(ctx, ctx->words[i].things);
 			ctx->words[i].things = ctx->malloc(ctx, sizeof(struct elem) * count);
 			memcpy(ctx->words[i].things, m, sizeof(struct elem) * count);
@@ -193,17 +287,10 @@ static int ink_add_indigenous(struct context* ctx, const char* name, struct elem
 			return i;
 		}
 	}
-	len = strlen(name);
-	copy = ctx->malloc(ctx, len+1);
-	if(copy == NULL) {
-		return -2;
-	}
-	memcpy(copy, name, len);
-	copy[len] = 0;
 	ctx->words[ctx->words_top].things = ctx->malloc(ctx, sizeof(struct elem) * count);
 	memcpy(ctx->words[ctx->words_top].things, m, sizeof(struct elem) * count);
 	ctx->words[ctx->words_top].size = count;
-	ctx->words[ctx->words_top].name = copy;
+	ctx->words[ctx->words_top].name = e;
 	return ctx->words_top++;
 }
 
@@ -336,11 +423,11 @@ int ink_destroy(struct context* ctx) {
 		/* ctx->inner_free(ctx, t->name); */
 	}
 	for (i = 0; i < ctx->words_top; ++i) {
-		ctx->inner_free(ctx, ctx->words[i].name);
+		ctx->words[i].name.type = INK_INTEGER;
 		ctx->free(ctx, ctx->words[i].things);
 	}
 	for (i = 0; i < ctx->native_words_top; ++i) {
-		ctx->inner_free(ctx, ctx->native_words[i].name);
+		ctx->native_words[i].name.type = INK_INTEGER;
 	}
 	for (i = 0; i < ctx->routines_top; ++i) {
 		if(ctx->routines[i].function_stack) ctx->free(ctx, ctx->routines[i].function_stack);
@@ -386,6 +473,7 @@ struct context* ink_make_context(void*(*malloc)(struct context*, size_t), void*(
 	ctx->lex_reserved_words_top = 0;
 	ctx->collections = 0;
 	ctx->steps = 0;
+	ink_new_type(ctx, "array", sizeof(struct ink_array), collect_array, gc_array);
 	return ctx;
 }
 
@@ -466,15 +554,15 @@ struct context* ink_make_default_context(void) {
 }
 #endif
 
-#ifndef NOSTRINGLITERALS
-static void new_protected_array(struct context* ctx);
-#endif
-
 static int ink_consume_one(int* end, struct context* pContext, char* r, int is_str) {
 	int i;
+	int tid;
 	int done;
 	struct elem value;
 	int err;
+	const char* name_it;
+	struct elem character;
+	struct ink_array ary;
 #ifndef NOSTRINGLITERALS
 	if(is_str) {
 		struct ink_routine* routine = pContext->routines + pContext->routine_current;
@@ -541,9 +629,20 @@ static int ink_consume_one(int* end, struct context* pContext, char* r, int is_s
 			return -19;
 		}
 	}
+	ary.elements = NULL;
+	ary.top = 0;
+	ary.capacity = 0;
+	ary.flags = 0;
+	tid =  get_type_by_name(pContext, "array");
+	name_it = r;
+	while (utf8_consume(&name_it, &character)) {
+		array_push_s(pContext, &ary, character);
+	}
+	ink_make_native(pContext, tid, &ary);
 	if (!done) {
 		for (i = 0; i < pContext->words_top; ++i) {
-			if (strcmp(r, pContext->words[i].name) == 0) {
+			struct ink_array* inner_name = ink_get_value(pContext, pContext->words[i].name);
+			if(ary.top == inner_name->top && memcmp(ary.elements, inner_name->elements, sizeof(struct elem)*ary.top) == 0) {
 				value.value = i;
 				value.type = INK_FUNCTION;
 				err = ink_push(pContext, value);
@@ -557,7 +656,8 @@ static int ink_consume_one(int* end, struct context* pContext, char* r, int is_s
 	}
 	if (!done) {
 		for (i = 0; i < pContext->native_words_top; ++i) {
-			if (strcmp(r, pContext->native_words[i].name) == 0) {
+			struct ink_array* inner_name = ink_get_value(pContext, pContext->native_words[i].name);
+			if(ary.top == inner_name->top && memcmp(ary.elements, inner_name->elements, sizeof(struct elem)*ary.top) == 0) {
 				value.value = i;
 				value.type = INK_NATIVE_FUNCTION;
 				err = ink_push(pContext, value);
@@ -1446,6 +1546,7 @@ void ink_gc(struct context* ctx) {
 	}
 	
 	for(i = 0; i < ctx->words_top; ++i) {
+		struct elem name;
 		struct fn* function = ctx->words + i;
 		for(j = 0; j < function->size; ++j) {
 			thing = function->things + j;
@@ -1454,6 +1555,16 @@ void ink_gc(struct context* ctx) {
 				++v->uses;
 			}
 		}
+		name = ctx->words[i].name;
+		ctx->types[name.type - 16].elements[name.value].in_use = 1;
+		ctx->types[name.type - 16].elements[name.value].uses = 1;
+	}
+
+	for(i = 0; i < ctx->native_words_top; ++i) {
+		struct elem name;
+		name = ctx->native_words[i].name;
+		ctx->types[name.type - 16].elements[name.value].in_use = 1;
+		ctx->types[name.type - 16].elements[name.value].uses = 1;
 	}
 	
 	/* Mark the rest of the data */
@@ -1520,20 +1631,14 @@ static void print_stacktrace(struct context* _) {
 		thing = currentRoutine->function_stack[i].executing;
 		switch(thing.type) {
 			case INK_NATIVE_FUNCTION: {
-				n = _->native_words[thing.value].name;
-				while (*n) {
-					_->putchar(_, *n);
-					++n;
-				}
+				ink_push(_, _->native_words[thing.value].name);
+				print_array_of_codepoints(_);
 				_->putchar(_, 10);
 				break;
 			}
 			case INK_FUNCTION:{
-				n = _->words[thing.value].name;
-				while (*n) {
-					_->putchar(_, *n);
-					++n;
-				}
+				ink_push(_, _->words[thing.value].name);
+				print_array_of_codepoints(_);
 				_->putchar(_, ':');
 				n = ink_itoa(_, currentRoutine->function_stack[i].index);
 				while (*n) {
@@ -2509,7 +2614,6 @@ int ink_std_library(struct context* ctx) {
 	v += ink_add_native(ctx, "int.xor", xor_int);
 #endif /* NOEXTRAARITHMETIC */
 #ifndef NOARRAYLIB
-	ink_new_type(ctx, "array", sizeof(struct ink_array), collect_array, gc_array);
 	ink_new_type(ctx, "array_marker", 0, collect_noop, gc_noop);
 	v += ink_add_native(ctx, "[", push_array_stack_delim);
 	v += ink_add_native(ctx, "]", push_delimited_array);

sh.c

@@ -55,30 +55,10 @@ static void resolve_word(struct context* ctx) {
     }
 #endif
     ink_pop(ctx);
-    it = 0;
-    name = ctx->malloc(ctx, ary->top*4);
-    for(i = 0; i < ary->top; ++i) {
-        a = ary->elements[i];
-        if(a.value <= 0x7F) {
-            name[it++] = a.value;
-        } else if(a.value <= 0x7FF) {
-            name[it++] = ((a.value & 0xFC0) >> 6) | 192;
-            name[it++] =(a.value & 0x3F) | 128;
-        } else if(a.value <= 0xFFFF) {
-            name[it++] = ((a.value & 0x3F000) >> 12) | 224;
-            name[it++] = ((a.value & 0xFC0) >> 6) | 128;
-            name[it++] = (a.value  & 0x3F) | 128;
-        } else if(a.value <= 0x10FFFF) {
-            name[it++] = ((a.value & 0x3C0000) >> 18)  | 240;
-            name[it++] = ((a.value & 0x3F000) >> 12) | 128;
-            name[it++] = ((a.value & 0xFC0) >> 6) | 128;
-            name[it++] = (a.value  & 0x3F) | 128;
-        }
-    }
-    name[it] = 0;
 
     for(nit = ctx->native_words; (!found) && nit != ctx->native_words + ctx->native_words_top; ++nit) {
-        if(strcmp(name, nit->name) == 0) {
+        struct ink_array* inner_name = ink_get_value(ctx, nit->name);
+        if(ary->top == inner_name->top && memcmp(ary->elements, inner_name->elements, sizeof(struct elem)*ary->top) == 0) {
             struct elem fn;
             fn.type = INK_NATIVE_FUNCTION;
             fn.value = nit - ctx->native_words;
@@ -87,7 +67,8 @@ static void resolve_word(struct context* ctx) {
         }
     }
     for(dit = ctx->words; (!found) && dit != ctx->words + ctx->words_top; ++dit) {
-        if(strcmp(name, dit->name) == 0) {
+        struct ink_array* inner_name = ink_get_value(ctx, dit->name);
+        if(ary->top == inner_name->top && memcmp(ary->elements, inner_name->elements, sizeof(struct elem)*ary->top) == 0) {
             struct elem fn;
             fn.type = INK_FUNCTION;
             fn.value = dit - ctx->words;
@@ -95,7 +76,6 @@ static void resolve_word(struct context* ctx) {
             found = 1;
         }
     }
-    ctx->free(ctx, name);
 }
 
 static void call_word(struct context* ctx) {