turn proc sub into a token with a region associated

3 files changed, 93 insertions, 45 deletions

diff --git a/lex.c b/lex.c
index 9e29420..d7641a4 100644
--- a/lex.c
+++ b/lex.c
@@ -50,11 +50,10 @@ MaybeChunk lex_expansion_chunk(RawLine* line, RegionVec* reg_arena, QuotationMod
             TokenVec inner_toks = lex_level(line, ')', reg_arena);
             expect_char(line, ')');
             expect_char(line, ')');
-            push_to_RegionVec(
+            Region* reg_ptr = push_to_RegionVec(
                 reg_arena, 
                 (Region){.mode = ARITHM, .tokens = inner_toks}
             );
-            Region* reg_ptr = &reg_arena->items[reg_arena->count];  // TODO: maybe have push() return a pointer [in the macro]
             return (MaybeChunk){
                 .some = true,
                 .value = (Chunk){
@@ -66,11 +65,10 @@ MaybeChunk lex_expansion_chunk(RawLine* line, RegionVec* reg_arena, QuotationMod
         } else {
             TokenVec inner_toks = lex_level(line, ')', reg_arena);
             expect_char(line, ')');
-            push_to_RegionVec(
+            Region* reg_ptr = push_to_RegionVec(
                 reg_arena, 
-                (Region){.mode = CMD_SUB, .tokens = inner_toks}
+                (Region){.mode = CMDS, .tokens = inner_toks}
             );
-            Region* reg_ptr = &reg_arena->items[reg_arena->count]; 
             return (MaybeChunk) {
                 .some = true,
                 .value = (Chunk){
@@ -164,65 +162,112 @@ Chunk collect_single_quoted_section(RawLine* line) {
 }
 
 
+typedef struct {
+    TokenVec tokens;
+    Word word_germ;
+    String unq_liter_germ;
+} LexState;
+
+
+void close_literal_germ(LexState* state) {
+   if (state->unq_liter_germ.count > 0) {
+        push_to_Word(
+            &state->word_germ, 
+            (Chunk){
+                .qmode = UN_Q, 
+                .kind = LITERAL, 
+                .as.literal_str = inner(state->unq_liter_germ)
+            }
+        );
+        state->unq_liter_germ = make_String(256);
+    } 
+}
+
+void close_word_and_lit_germs(LexState* state) {
+    close_literal_germ(state);
+    if (state->word_germ.count > 0) {
+        push_to_TokenVec(&state->tokens, (Token){.kind = TOK_WORD, .as.word = state->word_germ});
+        state->word_germ = make_Word(256);
+    }
+}
 
 TokenVec lex_level(RawLine* line, char until, RegionVec* reg_arena) {
 
-    printf("\n\n==== START LEVEL ===\n\n");
-    TokenVec tokens = make_TokenVec(256);    
-    Word curr_chunks = make_Word(256);
-    String word_buf = make_String(256);
+    LexState state = {
+        .tokens = make_TokenVec(256), 
+        .word_germ = make_Word(256), 
+        .unq_liter_germ = make_String(256)
+    };
 
     while (line->str[line->pos] != until) {
-        printf("\n\nLoop %d %d %d (%s)\n", tokens.count, curr_chunks.count, word_buf.count, word_buf.items);
 
         if (line->str[line->pos] == ' ') {
-            if (word_buf.count > 0) {
-                push_to_Word(
-                    &curr_chunks, 
-                    (Chunk){.qmode = UN_Q, .kind = LITERAL, .as.literal_str = inner(word_buf)}
-                );
-                word_buf = make_String(256);
-            }
-            if (curr_chunks.count > 0) {
-                push_to_TokenVec(&tokens, (Token){.kind = TOK_WORD, .as.word = curr_chunks});
-                curr_chunks = make_Word(256);
-            }
-
+            close_word_and_lit_germs(&state);
             while (line->str[line->pos] == ' ') line->pos++;
+            continue;
         }
 
         switch (line->str[line->pos]) {
             case '\'':
-                push_to_Word(&curr_chunks, collect_single_quoted_section(line));
+                push_to_Word(&state.word_germ, collect_single_quoted_section(line));
                 break;
             case '\"':
-                lex_double_quoted_section(line, &curr_chunks, reg_arena);
+                lex_double_quoted_section(line, &state.word_germ, reg_arena);
                 break;
             case '$':
                 line->pos++;
                 MaybeChunk mchunk = lex_expansion_chunk(line, reg_arena, UN_Q);
                 if (mchunk.some) {
-                    push_to_Word(&curr_chunks, mchunk.value);
+                    push_to_Word(&state.word_germ, mchunk.value);
                 } else {
-                    push_to_String(&word_buf, '$');
+                    push_to_String(&state.unq_liter_germ, '$');
+                }
+                break;
+            case '>':
+                close_word_and_lit_germs(&state);
+                line->pos++;
+
+                switch (line->str[line->pos]) {
+                    case '(':
+                        line->pos++;
+                        TokenVec sub_regtoks = lex_level(line, ')', reg_arena);
+                        Region subreg = {.mode = CMDS, .tokens = sub_regtoks};
+                        Region* reg_ptr = push_to_RegionVec(reg_arena, subreg);
+                        line->pos++;
+
+                        push_to_TokenVec(&state.tokens, (Token){.kind = TOK_PROCSUB_OUT, .as.procsub_region = reg_ptr});
+                        break;
+                    case '>':
+                        line->pos++;
+                        push_to_TokenVec(&state.tokens, (Token){.kind = TOK_SPECIAL, .as.spec = WRITE_REDIR_APPEND});
+                        break;
+                    default:
+                        push_to_TokenVec(&state.tokens, (Token){.kind = TOK_SPECIAL, .as.spec = WRITE_REDIR_TRUNC});
+                }
+                break;
+            case '<':
+                close_word_and_lit_germs(&state);
+                line->pos++;
+                if (line->str[line->pos] == '(') {
+                    line->pos++;
+                    TokenVec sub_regtoks = lex_level(line, ')', reg_arena);
+                    Region subreg = {.mode = CMDS, .tokens = sub_regtoks};
+                    Region* reg_ptr = push_to_RegionVec(reg_arena, subreg);
+                    line->pos++;
+
+                    push_to_TokenVec(&state.tokens, (Token){.kind = TOK_PROCSUB_IN, .as.procsub_region = reg_ptr});
+                    break;
+                } else {
+                        push_to_TokenVec(&state.tokens, (Token){.kind = TOK_SPECIAL, .as.spec = READ_REDIR});
                 }
                 break;
             default:
-                push_to_String(&word_buf, line->str[line->pos++]);
+                push_to_String(&state.unq_liter_germ, line->str[line->pos++]);
         }
     }
-
-    if (word_buf.count > 0) {
-        push_to_Word(
-            &curr_chunks, 
-            (Chunk){.qmode = UN_Q, .kind = LITERAL, .as.literal_str = inner(word_buf)}
-        );
-    }
-    if (curr_chunks.count > 0) {
-        push_to_TokenVec(&tokens, (Token){.kind = TOK_WORD, .as.word = curr_chunks});
-    }
-    printf("\nFIN: %d %d %d\n\n", tokens.count, curr_chunks.count, word_buf.count);
-    return tokens;
+    // TODO: handle until char _and_ EOL
+    close_word_and_lit_germs(&state);
+    return state.tokens;
 }
 
 
diff --git a/lex.h b/lex.h
index b6a102d..57c71bb 100644
--- a/lex.h
+++ b/lex.h
@@ -37,6 +37,8 @@ DECLARE_VEC(Word, WordVec)
 typedef enum {
     TOK_WORD,
     TOK_SPECIAL,
+    TOK_PROCSUB_IN,
+    TOK_PROCSUB_OUT,
 } TokenKind;
 
 typedef enum {
@@ -44,9 +46,6 @@ typedef enum {
     WRITE_REDIR_TRUNC,
     WRITE_REDIR_APPEND,
     READ_REDIR,
-    L_PROC_SUB_IN,
-    L_PROC_SUB_OUT,
-    PAREN,
     EOL,
 } SpecialToken;
 
@@ -56,6 +55,7 @@ typedef struct {
     union {
         Word word;
         SpecialToken spec; 
+        Region* procsub_region;
     } as;
 } Token;
     
@@ -65,7 +65,7 @@ DECLARE_VEC(Token, TokenVec)
 DECLARE_MAYBE(SpecialToken)  
 
 typedef enum {
-    CMD_SUB,
+    CMDS,
     ARITHM,
 } RegionMode;
 
diff --git a/typeutils.h b/typeutils.h
index fd285b2..d83217d 100644
--- a/typeutils.h
+++ b/typeutils.h
@@ -23,7 +23,7 @@
             .incr_step = step,                                      \
         };                                                          \
     }                                                               \
-    static inline void push_to_##Name(Name* vec, T item) {          \
+    static inline T* push_to_##Name(Name* vec, T item) {            \
         if (vec->count == vec->cap) {                               \
             vec->cap += vec->incr_step;                             \
             vec->items = realloc(                                   \
@@ -31,7 +31,10 @@
                 vec->cap * sizeof(T)                                \
             );                                                      \
         }                                                           \
-        vec->items[vec->count++] = item;                            \
+        vec->items[vec->count] = item;                              \
+        T* ptr = &vec->items[vec->count];                           \
+        vec->count++;                                               \
+        return ptr;                                                 \
     }