rewrite preproc for arenas

5 files changed, 166 insertions, 130 deletions

diff --git a/defs.c b/defs.c
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/defs.c
diff --git a/defs.h b/defs.h
new file mode 100644
index 0000000..c3bbd4b
--- /dev/null
+++ b/defs.h
@@ -0,0 +1,75 @@
+#pragma once
+
+#include "utils.h"
+
+
+typedef struct Region Region;
+
+
+// Lexing stage
+typedef enum {
+    UN_Q,
+    SINGLE_Q,
+    DOUBLE_Q,
+} QuotationMode;
+
+typedef enum {
+    LITERAL,
+    REGION_EXP,
+    VAR_EXP,    // TODO: Later include other exps
+} ChunkKind;
+
+typedef struct {
+    ChunkKind kind;
+    QuotationMode qmode;
+    union {
+        char* literal_str;
+        Region* reg_ptr;
+        char* varname;
+    } as;
+} Chunk;
+DECLARE_MAYBE(Chunk, MaybeChunk);
+DECLARE_VEC(Chunk*, Word)
+
+typedef enum {
+    TOK_WORD,
+    TOK_SPECIAL,
+    TOK_PROCSUB_IN,
+    TOK_PROCSUB_OUT,
+} TokenKind;
+
+typedef enum {
+    PIPE,
+    WRITE_REDIR_TRUNC,
+    WRITE_REDIR_APPEND,
+    READ_REDIR,
+    EOL,
+} SpecialToken;
+DECLARE_MAYBE(SpecialToken, MaybeSpecialToken)  
+
+typedef struct {
+    TokenKind kind;
+
+    union {
+        Word word;
+        SpecialToken spec; 
+        Region* procsub_region;
+    } as;
+} Token;
+
+
+DECLARE_VEC(Token*, TokenPtrVec)
+ 
+
+typedef enum {
+    CMDS,
+    ARITHM,
+} RegionMode;
+
+struct Region {
+    RegionMode mode;
+    TokenPtrVec tokens; 
+};
+DECLARE_VEC(Region, RegionVec)
+
+
diff --git a/lex.c b/lex.c
index 85b7120..5151119 100644
--- a/lex.c
+++ b/lex.c
@@ -7,6 +7,7 @@
 
 #include "lex.h"
 #include "utils.h"
+#include "defs.h"
 
 
 
@@ -26,7 +27,7 @@ String collect_identifier(RawLine* line) {
 
 
 
-TokenVec lex_level(RawLine* line, char until, RegionVec* reg_arena);
+TokenPtrVec lex_level(RawLine* line, char until, PreprocArena* arena);
 
 
 void expect_char(RawLine* line, char expected) {
@@ -37,6 +38,7 @@ void expect_char(RawLine* line, char expected) {
 }
 
 
+/*
 MaybeChunk lex_expansion_chunk(RawLine* line, RegionVec* reg_arena, QuotationMode qmode) {
     line->pos++;    // Consume $
     if (line->str[line->pos] == '(') {
@@ -92,9 +94,10 @@ MaybeChunk lex_expansion_chunk(RawLine* line, RegionVec* reg_arena, QuotationMod
     }
     return (MaybeChunk){.some = false};
 } 
+*/
 
 
-void lex_double_quoted_section(RawLine* line, Word* coll, RegionVec* reg_arena) {
+void lex_double_quoted_section(RawLine* line, Word* coll, PreprocArena* arena) {
     line->pos++;
     String accum = make_String(256);
     while (line->str[line->pos] != '\"') {
@@ -103,10 +106,13 @@ void lex_double_quoted_section(RawLine* line, Word* coll, RegionVec* reg_arena)
         MaybeChunk exp_chunk = {.some = false};
 
         if (line->str[line->pos] == '$') {
+            exit(1);
+            /*
             exp_chunk = lex_expansion_chunk(line, reg_arena, DOUBLE_Q);
             if (!exp_chunk.some) {
                     push_to_String(&accum, '$');
-            }            
+            } 
+            */
         } else {
             push_to_String(&accum, line->str[line->pos++]);
         }
@@ -114,24 +120,22 @@ void lex_double_quoted_section(RawLine* line, Word* coll, RegionVec* reg_arena)
         
         if (exp_chunk.some) {
             if (accum.count > 0) {
-                push_to_Word(coll, (Chunk){
-                    .qmode = DOUBLE_Q, 
-                    .kind = LITERAL, 
-                    .as.literal_str = inner(&accum)
-                });
+                Chunk accum_chunk = {.qmode = DOUBLE_Q, .kind = LITERAL, .as.literal_str = inner(&accum)};
+                Chunk* accum_ptr = push_to_ChunkVec(&arena->chunks, accum_chunk);
+                push_to_Word(coll, accum_ptr);
                 accum = make_String(256);
             }
-            push_to_Word(coll, exp_chunk.value);
+            Chunk* exp_ptr = push_to_ChunkVec(&arena->chunks, exp_chunk.value);
+            push_to_Word(coll, exp_ptr);
             exp_chunk.some = false;
         }
     }
     line->pos++;
     if (accum.count > 0) {
-        push_to_Word(coll, (Chunk){
-            .qmode = DOUBLE_Q, 
-            .kind = LITERAL, 
-            .as.literal_str = inner(&accum)
-        });
+        Chunk accum_chunk = {.qmode = DOUBLE_Q, .kind = LITERAL, .as.literal_str = inner(&accum)};
+        Chunk* accum_ptr = push_to_ChunkVec(&arena->chunks, accum_chunk);
+        push_to_Word(coll, accum_ptr);
+        accum = make_String(256);
     }
 }
 
@@ -162,64 +166,64 @@ Chunk collect_single_quoted_section(RawLine* line) {
 
 
 typedef struct {
-    TokenVec tokens;
     Word word_germ;
     String unq_liter_germ;
 } LexState;
 
 
-void close_literal_germ(LexState* state) {
+void close_word_and_lit_germs(LexState* state, TokenPtrVec* toks, PreprocArena* arena) {
    if (state->unq_liter_germ.count > 0) {
-        push_to_Word(
-            &state->word_germ, 
-            (Chunk){
-                .qmode = UN_Q, 
-                .kind = LITERAL, 
-                .as.literal_str = inner(&state->unq_liter_germ)
-            }
-        );
+        Chunk chu = {
+            .qmode = UN_Q, 
+            .kind = LITERAL, 
+            .as.literal_str = inner(&state->unq_liter_germ)
+        };
+        Chunk* chu_ptr = push_to_ChunkVec(&arena->chunks, chu);
+        push_to_Word(&state->word_germ, chu_ptr);
         state->unq_liter_germ = make_String(256);
-    } 
-}
-
-void close_word_and_lit_germs(LexState* state) {
-    close_literal_germ(state);
+    }
     if (state->word_germ.count > 0) {
-        push_to_TokenVec(&state->tokens, (Token){.kind = TOK_WORD, .as.word = state->word_germ});
+        Token tok = {.kind = TOK_WORD, .as.word = state->word_germ};
+        Token* tok_ptr = push_to_TokenVec(&arena->tokens, tok);
+        push_to_TokenPtrVec(toks, tok_ptr);
         state->word_germ = make_Word(256);
     }
 }
 
-TokenVec lex_level(RawLine* line, char until, RegionVec* reg_arena) {
+TokenPtrVec lex_level(RawLine* line, char until, PreprocArena* arena) {
 
+    TokenPtrVec line_toks = make_TokenPtrVec(256);
     LexState state = {
-        .tokens = make_TokenVec(256), 
         .word_germ = make_Word(256), 
         .unq_liter_germ = make_String(256)
     };
 
     while (line->str[line->pos] != until) {
 
-
         if (line->str[line->pos] == ' ') {
-            close_word_and_lit_germs(&state);
+            close_word_and_lit_germs(&state, &line_toks, arena);
             while (line->str[line->pos] == ' ') line->pos++;
             continue;
         }
 
         switch (line->str[line->pos]) {
             case '\'':
-                push_to_Word(&state.word_germ, collect_single_quoted_section(line));
+                Chunk sqchunk = collect_single_quoted_section(line);
+                Chunk* sqptr = push_to_ChunkVec(&arena->chunks, sqchunk);
+                push_to_Word(&state.word_germ, sqptr);
                 break;
             case '\"':
-                lex_double_quoted_section(line, &state.word_germ, reg_arena);
+                lex_double_quoted_section(line, &state.word_germ, arena);
                 break;
             case '|':
                 line->pos++;
-                close_word_and_lit_germs(&state);
-                push_to_TokenVec(&state.tokens, (Token){.kind = TOK_SPECIAL, .as.spec = PIPE});
+                close_word_and_lit_germs(&state, &line_toks, arena);
+                Token* tok_ptr = push_to_TokenVec(&arena->tokens, (Token){.kind = TOK_SPECIAL, .as.spec = PIPE});
+                push_to_TokenPtrVec(&line_toks, tok_ptr);
                 break;
             case '$':
+                exit(1);
+                /*
                 line->pos++;
                 MaybeChunk mchunk = lex_expansion_chunk(line, reg_arena, UN_Q);
                 if (mchunk.some) {
@@ -227,42 +231,48 @@ TokenVec lex_level(RawLine* line, char until, RegionVec* reg_arena) {
                 } else {
                     push_to_String(&state.unq_liter_germ, '$');
                 }
+                */
                 break;
             case '>':
-                close_word_and_lit_germs(&state);
+                close_word_and_lit_germs(&state, &line_toks, arena);
                 line->pos++;
 
                 switch (line->str[line->pos]) {
                     case '(':
                         line->pos++;
-                        TokenVec sub_regtoks = lex_level(line, ')', reg_arena);
-                        Region subreg = {.mode = CMDS, .tokens = sub_regtoks};
-                        Region* reg_ptr = push_to_RegionVec(reg_arena, subreg);
+                        TokenPtrVec sub_regtoks = lex_level(line, ')', arena);
                         line->pos++;
-
-                        push_to_TokenVec(&state.tokens, (Token){.kind = TOK_PROCSUB_OUT, .as.procsub_region = reg_ptr});
+                        Region subreg = {.mode = CMDS, .tokens = sub_regtoks};
+                        Region* subreg_ptr = push_to_RegionVec(&arena->regions, subreg);
+                        Token tok = {.kind = TOK_PROCSUB_OUT, .as.procsub_region = subreg_ptr};
+                        Token* tok_ptr = push_to_TokenVec(&arena->tokens, tok);
+                        push_to_TokenPtrVec(&line_toks, tok_ptr);
                         break;
                     case '>':
                         line->pos++;
-                        push_to_TokenVec(&state.tokens, (Token){.kind = TOK_SPECIAL, .as.spec = WRITE_REDIR_APPEND});
+                        Token* tok_ptr2 = push_to_TokenVec(&arena->tokens, (Token){.kind = TOK_SPECIAL, .as.spec = WRITE_REDIR_APPEND});
+                        push_to_TokenPtrVec(&line_toks, tok_ptr2);
                         break;
                     default:
-                        push_to_TokenVec(&state.tokens, (Token){.kind = TOK_SPECIAL, .as.spec = WRITE_REDIR_TRUNC});
+                        Token* tok_ptr3 = push_to_TokenVec(&arena->tokens, (Token){.kind = TOK_SPECIAL, .as.spec = WRITE_REDIR_TRUNC});
+                        push_to_TokenPtrVec(&line_toks, tok_ptr3);
                 }
                 break;
             case '<':
-                close_word_and_lit_germs(&state);
+                close_word_and_lit_germs(&state, &line_toks, arena);
                 line->pos++;
                 if (line->str[line->pos] == '(') {
                     line->pos++;
-                    TokenVec sub_regtoks = lex_level(line, ')', reg_arena);
-                    Region subreg = {.mode = CMDS, .tokens = sub_regtoks};
-                    Region* reg_ptr = push_to_RegionVec(reg_arena, subreg);
+                    TokenPtrVec sub_regtoks = lex_level(line, ')', arena);
                     line->pos++;
-
-                    push_to_TokenVec(&state.tokens, (Token){.kind = TOK_PROCSUB_IN, .as.procsub_region = reg_ptr});
+                    Region subreg = {.mode = CMDS, .tokens = sub_regtoks};
+                    Region* reg_ptr = push_to_RegionVec(&arena->regions, subreg);
+                    Token reg_tok = {.kind = TOK_PROCSUB_IN, .as.procsub_region = reg_ptr};
+                    Token* reg_tok_ptr = push_to_TokenVec(&arena->tokens, reg_tok);
+                    push_to_TokenPtrVec(&line_toks, reg_tok_ptr);
                 } else {
-                    push_to_TokenVec(&state.tokens, (Token){.kind = TOK_SPECIAL, .as.spec = READ_REDIR});
+                    Token* redir_ptr = push_to_TokenVec(&arena->tokens, (Token){.kind = TOK_SPECIAL, .as.spec = READ_REDIR});
+                    push_to_TokenPtrVec(&line_toks, redir_ptr);
                 }
                 break;
             default:
@@ -270,15 +280,15 @@ TokenVec lex_level(RawLine* line, char until, RegionVec* reg_arena) {
         }
     }
     // TODO: handle until char _and_ EOL
-    close_word_and_lit_germs(&state);
-    return state.tokens;
+    close_word_and_lit_germs(&state, &line_toks, arena);
+    return line_toks;
 }
 
 
-LexResult lex(char* input) {
+TokenPtrVec lex(char* input, PreprocArena* arena) {
     RawLine line = {.str = input, .pos = 0};
-    RegionVec reg_arena = make_RegionVec(256);
-    TokenVec toks = lex_level(&line, '\0', &reg_arena);
-    push_to_TokenVec(&toks, (Token){.kind = TOK_SPECIAL, .as.spec = EOL});
-    return (LexResult){.top_tokens = toks, .reg_arena = reg_arena};
+    TokenPtrVec toks = lex_level(&line, '\0', arena);
+    Token* eol_ptr = push_to_TokenVec(&arena->tokens, (Token){.kind = TOK_SPECIAL, .as.spec = EOL});
+    push_to_TokenPtrVec(&toks, eol_ptr);
+    return toks;
 }
diff --git a/lex.h b/lex.h
index 2f3cc8e..6c3c8b9 100644
--- a/lex.h
+++ b/lex.h
@@ -1,87 +1,30 @@
 #pragma once
 
+#include "defs.h"
 #include "utils.h"
 
-
-typedef struct Region Region;
-
-
-typedef enum {
-    UN_Q,
-    SINGLE_Q,
-    DOUBLE_Q,
-} QuotationMode;
-
-typedef enum {
-    LITERAL,
-    REGION_EXP,
-    VAR_EXP,    // TODO: Later include other exps
-} ChunkKind;
+DECLARE_VEC(Chunk, ChunkVec)
+DECLARE_VEC(Token, TokenVec)
 
 typedef struct {
-    ChunkKind kind;
-    QuotationMode qmode;
-    union {
-        char* literal_str;
-        Region* reg_ptr;
-        char* varname;
-    } as;
-} Chunk;
+    RegionVec regions;
+    TokenVec tokens;
+    ChunkVec chunks;
+} PreprocArena;
 
 
-DECLARE_VEC(Chunk, Word)
-
-DECLARE_VEC(Word, WordVec)
-
-typedef enum {
-    TOK_WORD,
-    TOK_SPECIAL,
-    TOK_PROCSUB_IN,
-    TOK_PROCSUB_OUT,
-} TokenKind;
-
-typedef enum {
-    PIPE,
-    WRITE_REDIR_TRUNC,
-    WRITE_REDIR_APPEND,
-    READ_REDIR,
-    EOL,
-} SpecialToken;
-
 typedef struct {
-    TokenKind kind;
-
-    union {
-        Word word;
-        SpecialToken spec; 
-        Region* procsub_region;
-    } as;
-} Token;
-    
-
-DECLARE_VEC(Token, TokenVec)
+    TokenVec top_tokens;
+    RegionVec reg_arena;
+} LexResult;
 
-DECLARE_MAYBE(SpecialToken, MaybeSpecialToken)  
 
-typedef enum {
-    CMDS,
-    ARITHM,
-} RegionMode;
+TokenPtrVec lex(char* input, PreprocArena* arena);
 
-struct Region {
-    RegionMode mode;
-    TokenVec tokens; 
-};
+   
 
-DECLARE_MAYBE(Chunk, MaybeChunk);
 
-DECLARE_VEC(Region, RegionVec)
 
 
-typedef struct {
-    TokenVec top_tokens;
-    RegionVec reg_arena;
-} LexResult;
 
 
-LexResult lex(char* input);
diff --git a/main.c b/main.c
index b64e889..a3fb4ec 100644
--- a/main.c
+++ b/main.c
@@ -3,6 +3,7 @@
 #include <readline/readline.h>
 #include <string.h>
 
+#include "defs.h"
 #include "lex.h"
 #include "parse.h"
 #include "exec.h"
@@ -15,9 +16,16 @@ int main() {
         line = readline("$ ");
         if (line[0] == '\0') continue;  
         if (strcmp(line, "exit") == 0) exit(0);
-        LexResult res = lex(line);
+        PreprocArena arena = {
+            .tokens = make_TokenVec(256),
+            .chunks = make_ChunkVec(256),
+            .regions = make_RegionVec(256),
+        };
+        TokenPtrVec line_toks = lex(line, &arena);
+        /* 
         Pipeline pl = parse_tokstream(res.top_tokens);
         run_pipeline(pl, &shstate);
+        */
     }
     free(line);
 }