omitted push-on-space logic added

1 files changed, 27 insertions, 6 deletions

diff --git a/lex.c b/lex.c
index c59057e..edcfd0b 100644
--- a/lex.c
+++ b/lex.c
@@ -30,7 +30,10 @@ TokenVec lex_level(RawLine* line, char until, ExpansionRegionVec* reg_arena);
 
 
 void expect_char(RawLine* line, char expected) {
-    if (line->str[line->pos] != expected) exit(42);     // TODO: nicer handling
+    if (line->str[line->pos] != expected) {
+        printf("Expected %c, got %c.", expected, line->str[line->pos]);
+        exit(45);     // TODO: nicer handling
+    }
     line->pos++; 
 }
 
@@ -38,7 +41,12 @@ void expect_char(RawLine* line, char expected) {
 MaybeChunk lex_expansion_chunk(RawLine* line, ExpansionRegionVec* reg_arena, QuotationMode qmode) {
     line->pos++;    // Consume $
     if (line->str[line->pos] == '(') {
+        line->pos++;
         if (line->str[line->pos] == '(') {
+            exit(1);    
+            // TODO: this (and potentially the other parsers must be custom, we need to recognize arithm-internal parens from the terminating `))` )
+        
+
             TokenVec inner_toks = lex_level(line, ')', reg_arena);
             expect_char(line, ')');
             expect_char(line, ')');
@@ -93,7 +101,7 @@ void lex_double_quoted_section(RawLine* line, ChunkVec* coll, ExpansionRegionVec
     line->pos++;
     String accum = make_String(256);
     while (line->str[line->pos] != '\"') {
-        if (line->str[line->pos] == '\0') exit(42);
+        if (line->str[line->pos] == '\0') exit(43);
 
         MaybeChunk exp_chunk = {.some = false};
 
@@ -144,7 +152,7 @@ Chunk collect_single_quoted_section(RawLine* line) {
     line->pos++;    // Consume quote mark
     String chunk_str = make_String(256);
     while (line->str[line->pos] != '\'') {
-        if (line->str[line->pos] == '\0') exit(42);
+        if (line->str[line->pos] == '\0') exit(44);
         push_to_String(&chunk_str, line->str[line->pos++]);
     }
     line->pos++;      // Consume closing quote
@@ -159,15 +167,28 @@ Chunk collect_single_quoted_section(RawLine* line) {
 
 TokenVec lex_level(RawLine* line, char until, ExpansionRegionVec* reg_arena) {
 
+    printf("\n\n==== START LEVEL ===\n\n");
     TokenVec tokens = make_TokenVec(256);    
     ChunkVec curr_chunks = make_ChunkVec(256);
     String word_buf = make_String(256);
 
     while (line->str[line->pos] != until) {
+        printf("\n\nLoop %d %d %d (%s)\n", tokens.count, curr_chunks.count, word_buf.count, word_buf.items);
 
         if (line->str[line->pos] == ' ') {
-            line->pos++;
-            continue;
+            if (word_buf.count > 0) {
+                push_to_ChunkVec(
+                    &curr_chunks, 
+                    (Chunk){.qmode = UN_Q, .kind = LITERAL, .as.literal_str = inner(word_buf)}
+                );
+                word_buf = make_String(256);
+            }
+            if (curr_chunks.count > 0) {
+                push_to_TokenVec(&tokens, (Token){.kind = TOK_WORD, .as.word = curr_chunks});
+                curr_chunks = make_ChunkVec(256);
+            }
+
+            while (line->str[line->pos] == ' ') line->pos++;
         }
 
         switch (line->str[line->pos]) {
@@ -191,7 +212,6 @@ TokenVec lex_level(RawLine* line, char until, ExpansionRegionVec* reg_arena) {
         }
     }
 
-    line->pos++;    // consume until char
     if (word_buf.count > 0) {
         push_to_ChunkVec(
             &curr_chunks, 
@@ -201,6 +221,7 @@ TokenVec lex_level(RawLine* line, char until, ExpansionRegionVec* reg_arena) {
     if (curr_chunks.count > 0) {
         push_to_TokenVec(&tokens, (Token){.kind = TOK_WORD, .as.word = curr_chunks});
     }
+    printf("\nFIN: %d %d %d\n\n", tokens.count, curr_chunks.count, word_buf.count);
     return tokens;
 }