From b4f4959d8401af0b68cc00ba199a6ef73cab21e5 Mon Sep 17 00:00:00 2001 From: Ákos Kőrösi Date: Mon, 18 May 2026 11:53:36 +0200 Subject: rewrite preproc for arenas --- defs.c | 0 defs.h | 75 +++++++++++++++++++++++++++++++++++++ lex.c | 130 +++++++++++++++++++++++++++++++++++------------------------------ lex.h | 81 ++++++---------------------------------- main.c | 10 ++++- 5 files changed, 166 insertions(+), 130 deletions(-) create mode 100644 defs.c create mode 100644 defs.h diff --git a/defs.c b/defs.c new file mode 100644 index 0000000..e69de29 diff --git a/defs.h b/defs.h new file mode 100644 index 0000000..c3bbd4b --- /dev/null +++ b/defs.h @@ -0,0 +1,75 @@ +#pragma once + +#include "utils.h" + + +typedef struct Region Region; + + +// Lexing stage +typedef enum { + UN_Q, + SINGLE_Q, + DOUBLE_Q, +} QuotationMode; + +typedef enum { + LITERAL, + REGION_EXP, + VAR_EXP, // TODO: Later include other exps +} ChunkKind; + +typedef struct { + ChunkKind kind; + QuotationMode qmode; + union { + char* literal_str; + Region* reg_ptr; + char* varname; + } as; +} Chunk; +DECLARE_MAYBE(Chunk, MaybeChunk); +DECLARE_VEC(Chunk*, Word) + +typedef enum { + TOK_WORD, + TOK_SPECIAL, + TOK_PROCSUB_IN, + TOK_PROCSUB_OUT, +} TokenKind; + +typedef enum { + PIPE, + WRITE_REDIR_TRUNC, + WRITE_REDIR_APPEND, + READ_REDIR, + EOL, +} SpecialToken; +DECLARE_MAYBE(SpecialToken, MaybeSpecialToken) + +typedef struct { + TokenKind kind; + + union { + Word word; + SpecialToken spec; + Region* procsub_region; + } as; +} Token; + + +DECLARE_VEC(Token*, TokenPtrVec) + + +typedef enum { + CMDS, + ARITHM, +} RegionMode; + +struct Region { + RegionMode mode; + TokenPtrVec tokens; +}; +DECLARE_VEC(Region, RegionVec) + + diff --git a/lex.c b/lex.c index 85b7120..5151119 100644 --- a/lex.c +++ b/lex.c @@ -7,6 +7,7 @@ #include "lex.h" #include "utils.h" +#include "defs.h" @@ -26,7 +27,7 @@ String collect_identifier(RawLine* line) { -TokenVec lex_level(RawLine* line, char until, RegionVec* reg_arena); +TokenPtrVec lex_level(RawLine* line, char until, PreprocArena* arena); void expect_char(RawLine* line, char expected) { @@ -37,6 +38,7 @@ void expect_char(RawLine* line, char expected) { } +/* MaybeChunk lex_expansion_chunk(RawLine* line, RegionVec* reg_arena, QuotationMode qmode) { line->pos++; // Consume $ if (line->str[line->pos] == '(') { @@ -92,9 +94,10 @@ MaybeChunk lex_expansion_chunk(RawLine* line, RegionVec* reg_arena, QuotationMod } return (MaybeChunk){.some = false}; } +*/ -void lex_double_quoted_section(RawLine* line, Word* coll, RegionVec* reg_arena) { +void lex_double_quoted_section(RawLine* line, Word* coll, PreprocArena* arena) { line->pos++; String accum = make_String(256); while (line->str[line->pos] != '\"') { @@ -103,10 +106,13 @@ void lex_double_quoted_section(RawLine* line, Word* coll, RegionVec* reg_arena) MaybeChunk exp_chunk = {.some = false}; if (line->str[line->pos] == '$') { + exit(1); + /* exp_chunk = lex_expansion_chunk(line, reg_arena, DOUBLE_Q); if (!exp_chunk.some) { push_to_String(&accum, '$'); - } + } + */ } else { push_to_String(&accum, line->str[line->pos++]); } @@ -114,24 +120,22 @@ void lex_double_quoted_section(RawLine* line, Word* coll, RegionVec* reg_arena) if (exp_chunk.some) { if (accum.count > 0) { - push_to_Word(coll, (Chunk){ - .qmode = DOUBLE_Q, - .kind = LITERAL, - .as.literal_str = inner(&accum) - }); + Chunk accum_chunk = {.qmode = DOUBLE_Q, .kind = LITERAL, .as.literal_str = inner(&accum)}; + Chunk* accum_ptr = push_to_ChunkVec(&arena->chunks, accum_chunk); + push_to_Word(coll, accum_ptr); accum = make_String(256); } - push_to_Word(coll, exp_chunk.value); + Chunk* exp_ptr = push_to_ChunkVec(&arena->chunks, exp_chunk.value); + push_to_Word(coll, exp_ptr); exp_chunk.some = false; } } line->pos++; if (accum.count > 0) { - push_to_Word(coll, (Chunk){ - .qmode = DOUBLE_Q, - .kind = LITERAL, - .as.literal_str = inner(&accum) - }); + Chunk accum_chunk = {.qmode = DOUBLE_Q, .kind = LITERAL, .as.literal_str = inner(&accum)}; + Chunk* accum_ptr = push_to_ChunkVec(&arena->chunks, accum_chunk); + push_to_Word(coll, accum_ptr); + accum = make_String(256); } } @@ -162,64 +166,64 @@ Chunk collect_single_quoted_section(RawLine* line) { typedef struct { - TokenVec tokens; Word word_germ; String unq_liter_germ; } LexState; -void close_literal_germ(LexState* state) { +void close_word_and_lit_germs(LexState* state, TokenPtrVec* toks, PreprocArena* arena) { if (state->unq_liter_germ.count > 0) { - push_to_Word( - &state->word_germ, - (Chunk){ - .qmode = UN_Q, - .kind = LITERAL, - .as.literal_str = inner(&state->unq_liter_germ) - } - ); + Chunk chu = { + .qmode = UN_Q, + .kind = LITERAL, + .as.literal_str = inner(&state->unq_liter_germ) + }; + Chunk* chu_ptr = push_to_ChunkVec(&arena->chunks, chu); + push_to_Word(&state->word_germ, chu_ptr); state->unq_liter_germ = make_String(256); - } -} - -void close_word_and_lit_germs(LexState* state) { - close_literal_germ(state); + } if (state->word_germ.count > 0) { - push_to_TokenVec(&state->tokens, (Token){.kind = TOK_WORD, .as.word = state->word_germ}); + Token tok = {.kind = TOK_WORD, .as.word = state->word_germ}; + Token* tok_ptr = push_to_TokenVec(&arena->tokens, tok); + push_to_TokenPtrVec(toks, tok_ptr); state->word_germ = make_Word(256); } } -TokenVec lex_level(RawLine* line, char until, RegionVec* reg_arena) { +TokenPtrVec lex_level(RawLine* line, char until, PreprocArena* arena) { + TokenPtrVec line_toks = make_TokenPtrVec(256); LexState state = { - .tokens = make_TokenVec(256), .word_germ = make_Word(256), .unq_liter_germ = make_String(256) }; while (line->str[line->pos] != until) { - if (line->str[line->pos] == ' ') { - close_word_and_lit_germs(&state); + close_word_and_lit_germs(&state, &line_toks, arena); while (line->str[line->pos] == ' ') line->pos++; continue; } switch (line->str[line->pos]) { case '\'': - push_to_Word(&state.word_germ, collect_single_quoted_section(line)); + Chunk sqchunk = collect_single_quoted_section(line); + Chunk* sqptr = push_to_ChunkVec(&arena->chunks, sqchunk); + push_to_Word(&state.word_germ, sqptr); break; case '\"': - lex_double_quoted_section(line, &state.word_germ, reg_arena); + lex_double_quoted_section(line, &state.word_germ, arena); break; case '|': line->pos++; - close_word_and_lit_germs(&state); - push_to_TokenVec(&state.tokens, (Token){.kind = TOK_SPECIAL, .as.spec = PIPE}); + close_word_and_lit_germs(&state, &line_toks, arena); + Token* tok_ptr = push_to_TokenVec(&arena->tokens, (Token){.kind = TOK_SPECIAL, .as.spec = PIPE}); + push_to_TokenPtrVec(&line_toks, tok_ptr); break; case '$': + exit(1); + /* line->pos++; MaybeChunk mchunk = lex_expansion_chunk(line, reg_arena, UN_Q); if (mchunk.some) { @@ -227,42 +231,48 @@ TokenVec lex_level(RawLine* line, char until, RegionVec* reg_arena) { } else { push_to_String(&state.unq_liter_germ, '$'); } + */ break; case '>': - close_word_and_lit_germs(&state); + close_word_and_lit_germs(&state, &line_toks, arena); line->pos++; switch (line->str[line->pos]) { case '(': line->pos++; - TokenVec sub_regtoks = lex_level(line, ')', reg_arena); - Region subreg = {.mode = CMDS, .tokens = sub_regtoks}; - Region* reg_ptr = push_to_RegionVec(reg_arena, subreg); + TokenPtrVec sub_regtoks = lex_level(line, ')', arena); line->pos++; - - push_to_TokenVec(&state.tokens, (Token){.kind = TOK_PROCSUB_OUT, .as.procsub_region = reg_ptr}); + Region subreg = {.mode = CMDS, .tokens = sub_regtoks}; + Region* subreg_ptr = push_to_RegionVec(&arena->regions, subreg); + Token tok = {.kind = TOK_PROCSUB_OUT, .as.procsub_region = subreg_ptr}; + Token* tok_ptr = push_to_TokenVec(&arena->tokens, tok); + push_to_TokenPtrVec(&line_toks, tok_ptr); break; case '>': line->pos++; - push_to_TokenVec(&state.tokens, (Token){.kind = TOK_SPECIAL, .as.spec = WRITE_REDIR_APPEND}); + Token* tok_ptr2 = push_to_TokenVec(&arena->tokens, (Token){.kind = TOK_SPECIAL, .as.spec = WRITE_REDIR_APPEND}); + push_to_TokenPtrVec(&line_toks, tok_ptr2); break; default: - push_to_TokenVec(&state.tokens, (Token){.kind = TOK_SPECIAL, .as.spec = WRITE_REDIR_TRUNC}); + Token* tok_ptr3 = push_to_TokenVec(&arena->tokens, (Token){.kind = TOK_SPECIAL, .as.spec = WRITE_REDIR_TRUNC}); + push_to_TokenPtrVec(&line_toks, tok_ptr3); } break; case '<': - close_word_and_lit_germs(&state); + close_word_and_lit_germs(&state, &line_toks, arena); line->pos++; if (line->str[line->pos] == '(') { line->pos++; - TokenVec sub_regtoks = lex_level(line, ')', reg_arena); - Region subreg = {.mode = CMDS, .tokens = sub_regtoks}; - Region* reg_ptr = push_to_RegionVec(reg_arena, subreg); + TokenPtrVec sub_regtoks = lex_level(line, ')', arena); line->pos++; - - push_to_TokenVec(&state.tokens, (Token){.kind = TOK_PROCSUB_IN, .as.procsub_region = reg_ptr}); + Region subreg = {.mode = CMDS, .tokens = sub_regtoks}; + Region* reg_ptr = push_to_RegionVec(&arena->regions, subreg); + Token reg_tok = {.kind = TOK_PROCSUB_IN, .as.procsub_region = reg_ptr}; + Token* reg_tok_ptr = push_to_TokenVec(&arena->tokens, reg_tok); + push_to_TokenPtrVec(&line_toks, reg_tok_ptr); } else { - push_to_TokenVec(&state.tokens, (Token){.kind = TOK_SPECIAL, .as.spec = READ_REDIR}); + Token* redir_ptr = push_to_TokenVec(&arena->tokens, (Token){.kind = TOK_SPECIAL, .as.spec = READ_REDIR}); + push_to_TokenPtrVec(&line_toks, redir_ptr); } break; default: @@ -270,15 +280,15 @@ TokenVec lex_level(RawLine* line, char until, RegionVec* reg_arena) { } } // TODO: handle until char _and_ EOL - close_word_and_lit_germs(&state); - return state.tokens; + close_word_and_lit_germs(&state, &line_toks, arena); + return line_toks; } -LexResult lex(char* input) { +TokenPtrVec lex(char* input, PreprocArena* arena) { RawLine line = {.str = input, .pos = 0}; - RegionVec reg_arena = make_RegionVec(256); - TokenVec toks = lex_level(&line, '\0', ®_arena); - push_to_TokenVec(&toks, (Token){.kind = TOK_SPECIAL, .as.spec = EOL}); - return (LexResult){.top_tokens = toks, .reg_arena = reg_arena}; + TokenPtrVec toks = lex_level(&line, '\0', arena); + Token* eol_ptr = push_to_TokenVec(&arena->tokens, (Token){.kind = TOK_SPECIAL, .as.spec = EOL}); + push_to_TokenPtrVec(&toks, eol_ptr); + return toks; } diff --git a/lex.h b/lex.h index 2f3cc8e..6c3c8b9 100644 --- a/lex.h +++ b/lex.h @@ -1,87 +1,30 @@ #pragma once +#include "defs.h" #include "utils.h" - -typedef struct Region Region; - - -typedef enum { - UN_Q, - SINGLE_Q, - DOUBLE_Q, -} QuotationMode; - -typedef enum { - LITERAL, - REGION_EXP, - VAR_EXP, // TODO: Later include other exps -} ChunkKind; +DECLARE_VEC(Chunk, ChunkVec) +DECLARE_VEC(Token, TokenVec) typedef struct { - ChunkKind kind; - QuotationMode qmode; - union { - char* literal_str; - Region* reg_ptr; - char* varname; - } as; -} Chunk; + RegionVec regions; + TokenVec tokens; + ChunkVec chunks; +} PreprocArena; -DECLARE_VEC(Chunk, Word) - -DECLARE_VEC(Word, WordVec) - -typedef enum { - TOK_WORD, - TOK_SPECIAL, - TOK_PROCSUB_IN, - TOK_PROCSUB_OUT, -} TokenKind; - -typedef enum { - PIPE, - WRITE_REDIR_TRUNC, - WRITE_REDIR_APPEND, - READ_REDIR, - EOL, -} SpecialToken; - typedef struct { - TokenKind kind; - - union { - Word word; - SpecialToken spec; - Region* procsub_region; - } as; -} Token; - - -DECLARE_VEC(Token, TokenVec) + TokenVec top_tokens; + RegionVec reg_arena; +} LexResult; -DECLARE_MAYBE(SpecialToken, MaybeSpecialToken) -typedef enum { - CMDS, - ARITHM, -} RegionMode; +TokenPtrVec lex(char* input, PreprocArena* arena); -struct Region { - RegionMode mode; - TokenVec tokens; -}; + -DECLARE_MAYBE(Chunk, MaybeChunk); -DECLARE_VEC(Region, RegionVec) -typedef struct { - TokenVec top_tokens; - RegionVec reg_arena; -} LexResult; -LexResult lex(char* input); diff --git a/main.c b/main.c index b64e889..a3fb4ec 100644 --- a/main.c +++ b/main.c @@ -3,6 +3,7 @@ #include #include +#include "defs.h" #include "lex.h" #include "parse.h" #include "exec.h" @@ -15,9 +16,16 @@ int main() { line = readline("$ "); if (line[0] == '\0') continue; if (strcmp(line, "exit") == 0) exit(0); - LexResult res = lex(line); + PreprocArena arena = { + .tokens = make_TokenVec(256), + .chunks = make_ChunkVec(256), + .regions = make_RegionVec(256), + }; + TokenPtrVec line_toks = lex(line, &arena); + /* Pipeline pl = parse_tokstream(res.top_tokens); run_pipeline(pl, &shstate); + */ } free(line); } -- cgit v1.2.3