rewrite preproc for arenas

1 files changed, 70 insertions, 60 deletions

diff --git a/lex.c b/lex.c
index 85b7120..5151119 100644
--- a/lex.c
+++ b/lex.c
@@ -7,6 +7,7 @@
 
 #include "lex.h"
 #include "utils.h"
+#include "defs.h"
 
 
 
@@ -26,7 +27,7 @@ String collect_identifier(RawLine* line) {
 
 
 
-TokenVec lex_level(RawLine* line, char until, RegionVec* reg_arena);
+TokenPtrVec lex_level(RawLine* line, char until, PreprocArena* arena);
 
 
 void expect_char(RawLine* line, char expected) {
@@ -37,6 +38,7 @@ void expect_char(RawLine* line, char expected) {
 }
 
 
+/*
 MaybeChunk lex_expansion_chunk(RawLine* line, RegionVec* reg_arena, QuotationMode qmode) {
     line->pos++;    // Consume $
     if (line->str[line->pos] == '(') {
@@ -92,9 +94,10 @@ MaybeChunk lex_expansion_chunk(RawLine* line, RegionVec* reg_arena, QuotationMod
     }
     return (MaybeChunk){.some = false};
 } 
+*/
 
 
-void lex_double_quoted_section(RawLine* line, Word* coll, RegionVec* reg_arena) {
+void lex_double_quoted_section(RawLine* line, Word* coll, PreprocArena* arena) {
     line->pos++;
     String accum = make_String(256);
     while (line->str[line->pos] != '\"') {
@@ -103,10 +106,13 @@ void lex_double_quoted_section(RawLine* line, Word* coll, RegionVec* reg_arena)
         MaybeChunk exp_chunk = {.some = false};
 
         if (line->str[line->pos] == '$') {
+            exit(1);
+            /*
             exp_chunk = lex_expansion_chunk(line, reg_arena, DOUBLE_Q);
             if (!exp_chunk.some) {
                     push_to_String(&accum, '$');
-            }            
+            } 
+            */
         } else {
             push_to_String(&accum, line->str[line->pos++]);
         }
@@ -114,24 +120,22 @@ void lex_double_quoted_section(RawLine* line, Word* coll, RegionVec* reg_arena)
         
         if (exp_chunk.some) {
             if (accum.count > 0) {
-                push_to_Word(coll, (Chunk){
-                    .qmode = DOUBLE_Q, 
-                    .kind = LITERAL, 
-                    .as.literal_str = inner(&accum)
-                });
+                Chunk accum_chunk = {.qmode = DOUBLE_Q, .kind = LITERAL, .as.literal_str = inner(&accum)};
+                Chunk* accum_ptr = push_to_ChunkVec(&arena->chunks, accum_chunk);
+                push_to_Word(coll, accum_ptr);
                 accum = make_String(256);
             }
-            push_to_Word(coll, exp_chunk.value);
+            Chunk* exp_ptr = push_to_ChunkVec(&arena->chunks, exp_chunk.value);
+            push_to_Word(coll, exp_ptr);
             exp_chunk.some = false;
         }
     }
     line->pos++;
     if (accum.count > 0) {
-        push_to_Word(coll, (Chunk){
-            .qmode = DOUBLE_Q, 
-            .kind = LITERAL, 
-            .as.literal_str = inner(&accum)
-        });
+        Chunk accum_chunk = {.qmode = DOUBLE_Q, .kind = LITERAL, .as.literal_str = inner(&accum)};
+        Chunk* accum_ptr = push_to_ChunkVec(&arena->chunks, accum_chunk);
+        push_to_Word(coll, accum_ptr);
+        accum = make_String(256);
     }
 }
 
@@ -162,64 +166,64 @@ Chunk collect_single_quoted_section(RawLine* line) {
 
 
 typedef struct {
-    TokenVec tokens;
     Word word_germ;
     String unq_liter_germ;
 } LexState;
 
 
-void close_literal_germ(LexState* state) {
+void close_word_and_lit_germs(LexState* state, TokenPtrVec* toks, PreprocArena* arena) {
    if (state->unq_liter_germ.count > 0) {
-        push_to_Word(
-            &state->word_germ, 
-            (Chunk){
-                .qmode = UN_Q, 
-                .kind = LITERAL, 
-                .as.literal_str = inner(&state->unq_liter_germ)
-            }
-        );
+        Chunk chu = {
+            .qmode = UN_Q, 
+            .kind = LITERAL, 
+            .as.literal_str = inner(&state->unq_liter_germ)
+        };
+        Chunk* chu_ptr = push_to_ChunkVec(&arena->chunks, chu);
+        push_to_Word(&state->word_germ, chu_ptr);
         state->unq_liter_germ = make_String(256);
-    } 
-}
-
-void close_word_and_lit_germs(LexState* state) {
-    close_literal_germ(state);
+    }
     if (state->word_germ.count > 0) {
-        push_to_TokenVec(&state->tokens, (Token){.kind = TOK_WORD, .as.word = state->word_germ});
+        Token tok = {.kind = TOK_WORD, .as.word = state->word_germ};
+        Token* tok_ptr = push_to_TokenVec(&arena->tokens, tok);
+        push_to_TokenPtrVec(toks, tok_ptr);
         state->word_germ = make_Word(256);
     }
 }
 
-TokenVec lex_level(RawLine* line, char until, RegionVec* reg_arena) {
+TokenPtrVec lex_level(RawLine* line, char until, PreprocArena* arena) {
 
+    TokenPtrVec line_toks = make_TokenPtrVec(256);
     LexState state = {
-        .tokens = make_TokenVec(256), 
         .word_germ = make_Word(256), 
         .unq_liter_germ = make_String(256)
     };
 
     while (line->str[line->pos] != until) {
 
-
         if (line->str[line->pos] == ' ') {
-            close_word_and_lit_germs(&state);
+            close_word_and_lit_germs(&state, &line_toks, arena);
             while (line->str[line->pos] == ' ') line->pos++;
             continue;
         }
 
         switch (line->str[line->pos]) {
             case '\'':
-                push_to_Word(&state.word_germ, collect_single_quoted_section(line));
+                Chunk sqchunk = collect_single_quoted_section(line);
+                Chunk* sqptr = push_to_ChunkVec(&arena->chunks, sqchunk);
+                push_to_Word(&state.word_germ, sqptr);
                 break;
             case '\"':
-                lex_double_quoted_section(line, &state.word_germ, reg_arena);
+                lex_double_quoted_section(line, &state.word_germ, arena);
                 break;
             case '|':
                 line->pos++;
-                close_word_and_lit_germs(&state);
-                push_to_TokenVec(&state.tokens, (Token){.kind = TOK_SPECIAL, .as.spec = PIPE});
+                close_word_and_lit_germs(&state, &line_toks, arena);
+                Token* tok_ptr = push_to_TokenVec(&arena->tokens, (Token){.kind = TOK_SPECIAL, .as.spec = PIPE});
+                push_to_TokenPtrVec(&line_toks, tok_ptr);
                 break;
             case '$':
+                exit(1);
+                /*
                 line->pos++;
                 MaybeChunk mchunk = lex_expansion_chunk(line, reg_arena, UN_Q);
                 if (mchunk.some) {
@@ -227,42 +231,48 @@ TokenVec lex_level(RawLine* line, char until, RegionVec* reg_arena) {
                 } else {
                     push_to_String(&state.unq_liter_germ, '$');
                 }
+                */
                 break;
             case '>':
-                close_word_and_lit_germs(&state);
+                close_word_and_lit_germs(&state, &line_toks, arena);
                 line->pos++;
 
                 switch (line->str[line->pos]) {
                     case '(':
                         line->pos++;
-                        TokenVec sub_regtoks = lex_level(line, ')', reg_arena);
-                        Region subreg = {.mode = CMDS, .tokens = sub_regtoks};
-                        Region* reg_ptr = push_to_RegionVec(reg_arena, subreg);
+                        TokenPtrVec sub_regtoks = lex_level(line, ')', arena);
                         line->pos++;
-
-                        push_to_TokenVec(&state.tokens, (Token){.kind = TOK_PROCSUB_OUT, .as.procsub_region = reg_ptr});
+                        Region subreg = {.mode = CMDS, .tokens = sub_regtoks};
+                        Region* subreg_ptr = push_to_RegionVec(&arena->regions, subreg);
+                        Token tok = {.kind = TOK_PROCSUB_OUT, .as.procsub_region = subreg_ptr};
+                        Token* tok_ptr = push_to_TokenVec(&arena->tokens, tok);
+                        push_to_TokenPtrVec(&line_toks, tok_ptr);
                         break;
                     case '>':
                         line->pos++;
-                        push_to_TokenVec(&state.tokens, (Token){.kind = TOK_SPECIAL, .as.spec = WRITE_REDIR_APPEND});
+                        Token* tok_ptr2 = push_to_TokenVec(&arena->tokens, (Token){.kind = TOK_SPECIAL, .as.spec = WRITE_REDIR_APPEND});
+                        push_to_TokenPtrVec(&line_toks, tok_ptr2);
                         break;
                     default:
-                        push_to_TokenVec(&state.tokens, (Token){.kind = TOK_SPECIAL, .as.spec = WRITE_REDIR_TRUNC});
+                        Token* tok_ptr3 = push_to_TokenVec(&arena->tokens, (Token){.kind = TOK_SPECIAL, .as.spec = WRITE_REDIR_TRUNC});
+                        push_to_TokenPtrVec(&line_toks, tok_ptr3);
                 }
                 break;
             case '<':
-                close_word_and_lit_germs(&state);
+                close_word_and_lit_germs(&state, &line_toks, arena);
                 line->pos++;
                 if (line->str[line->pos] == '(') {
                     line->pos++;
-                    TokenVec sub_regtoks = lex_level(line, ')', reg_arena);
-                    Region subreg = {.mode = CMDS, .tokens = sub_regtoks};
-                    Region* reg_ptr = push_to_RegionVec(reg_arena, subreg);
+                    TokenPtrVec sub_regtoks = lex_level(line, ')', arena);
                     line->pos++;
-
-                    push_to_TokenVec(&state.tokens, (Token){.kind = TOK_PROCSUB_IN, .as.procsub_region = reg_ptr});
+                    Region subreg = {.mode = CMDS, .tokens = sub_regtoks};
+                    Region* reg_ptr = push_to_RegionVec(&arena->regions, subreg);
+                    Token reg_tok = {.kind = TOK_PROCSUB_IN, .as.procsub_region = reg_ptr};
+                    Token* reg_tok_ptr = push_to_TokenVec(&arena->tokens, reg_tok);
+                    push_to_TokenPtrVec(&line_toks, reg_tok_ptr);
                 } else {
-                    push_to_TokenVec(&state.tokens, (Token){.kind = TOK_SPECIAL, .as.spec = READ_REDIR});
+                    Token* redir_ptr = push_to_TokenVec(&arena->tokens, (Token){.kind = TOK_SPECIAL, .as.spec = READ_REDIR});
+                    push_to_TokenPtrVec(&line_toks, redir_ptr);
                 }
                 break;
             default:
@@ -270,15 +280,15 @@ TokenVec lex_level(RawLine* line, char until, RegionVec* reg_arena) {
         }
     }
     // TODO: handle until char _and_ EOL
-    close_word_and_lit_germs(&state);
-    return state.tokens;
+    close_word_and_lit_germs(&state, &line_toks, arena);
+    return line_toks;
 }
 
 
-LexResult lex(char* input) {
+TokenPtrVec lex(char* input, PreprocArena* arena) {
     RawLine line = {.str = input, .pos = 0};
-    RegionVec reg_arena = make_RegionVec(256);
-    TokenVec toks = lex_level(&line, '\0', &reg_arena);
-    push_to_TokenVec(&toks, (Token){.kind = TOK_SPECIAL, .as.spec = EOL});
-    return (LexResult){.top_tokens = toks, .reg_arena = reg_arena};
+    TokenPtrVec toks = lex_level(&line, '\0', arena);
+    Token* eol_ptr = push_to_TokenVec(&arena->tokens, (Token){.kind = TOK_SPECIAL, .as.spec = EOL});
+    push_to_TokenPtrVec(&toks, eol_ptr);
+    return toks;
 }