#include #include #include #include #include #include #include "lex.h" #include "utils.h" #include "defs.h" typedef struct { char* str; size_t pos; } RawLine; String collect_identifier(RawLine* line) { String id_buf = make_String(256); while (line->str[line->pos] == '_' || isalnum(line->str[line->pos])) { push_to_String(&id_buf, line->str[line->pos++]); } return id_buf; } TokenPtrVec lex_level(RawLine* line, char until, PreprocArena* arena); void expect_char(RawLine* line, char expected) { if (line->str[line->pos] != expected) { exit(45); // TODO: nicer handling } line->pos++; } /* MaybeChunk lex_expansion_chunk(RawLine* line, RegionVec* reg_arena, QuotationMode qmode) { line->pos++; // Consume $ if (line->str[line->pos] == '(') { line->pos++; if (line->str[line->pos] == '(') { exit(1); // TODO: this (and potentially the other parsers must be custom, we need to recognize arithm-internal parens from the terminating `))` ) TokenVec inner_toks = lex_level(line, ')', reg_arena); expect_char(line, ')'); expect_char(line, ')'); Region* reg_ptr = push_to_RegionVec( reg_arena, (Region){.mode = ARITHM, .tokens = inner_toks} ); return (MaybeChunk){ .some = true, .value = (Chunk){ .qmode = qmode, .kind = REGION_EXP, .as.reg_ptr = reg_ptr }, }; } else { TokenVec inner_toks = lex_level(line, ')', reg_arena); expect_char(line, ')'); Region* reg_ptr = push_to_RegionVec( reg_arena, (Region){.mode = CMDS, .tokens = inner_toks} ); return (MaybeChunk) { .some = true, .value = (Chunk){ .qmode = qmode, .kind = REGION_EXP, .as.reg_ptr = reg_ptr, } }; } } else { String id_name = collect_identifier(line); if (id_name.count > 0) { return (MaybeChunk){ .some = true, .value = (Chunk){ .qmode = qmode, .kind = VAR_EXP, .as.varname = inner(&id_name), } }; } } return (MaybeChunk){.some = false}; } */ void lex_double_quoted_section(RawLine* line, Word* coll, PreprocArena* arena) { line->pos++; String accum = make_String(256); while (line->str[line->pos] != '\"') { if (line->str[line->pos] == '\0') exit(43); MaybeChunk exp_chunk = {.some = false}; if (line->str[line->pos] == '$') { exit(1); /* exp_chunk = lex_expansion_chunk(line, reg_arena, DOUBLE_Q); if (!exp_chunk.some) { push_to_String(&accum, '$'); } */ } else { push_to_String(&accum, line->str[line->pos++]); } // TODO: other special cases beyond $ if (exp_chunk.some) { if (accum.count > 0) { Chunk accum_chunk = {.qmode = DOUBLE_Q, .kind = LITERAL, .as.literal_str = inner(&accum)}; Chunk* accum_ptr = push_to_ChunkVec(&arena->chunks, accum_chunk); push_to_Word(coll, accum_ptr); accum = make_String(256); } Chunk* exp_ptr = push_to_ChunkVec(&arena->chunks, exp_chunk.value); push_to_Word(coll, exp_ptr); exp_chunk.some = false; } } line->pos++; if (accum.count > 0) { Chunk accum_chunk = {.qmode = DOUBLE_Q, .kind = LITERAL, .as.literal_str = inner(&accum)}; Chunk* accum_ptr = push_to_ChunkVec(&arena->chunks, accum_chunk); push_to_Word(coll, accum_ptr); accum = make_String(256); } } Chunk next_word_unquoted(RawLine* line, RegionVec* reg_arena) { String buf = make_String(256); for (char c; (c=line->str[line->pos], c != '$' && c != '\0' && c != ' '); line->pos++) { push_to_String(&buf, c); } return (Chunk){.qmode = UN_Q, .kind = LITERAL, .as.literal_str = inner(&buf)}; } Chunk collect_single_quoted_section(RawLine* line) { line->pos++; // Consume quote mark String chunk_str = make_String(256); while (line->str[line->pos] != '\'') { if (line->str[line->pos] == '\0') exit(44); push_to_String(&chunk_str, line->str[line->pos++]); } line->pos++; // Consume closing quote return (Chunk){ .qmode = SINGLE_Q, .kind = LITERAL, .as.literal_str = inner(&chunk_str), }; } typedef struct { Word word_germ; String unq_liter_germ; } LexState; void close_word_and_lit_germs(LexState* state, TokenPtrVec* toks, PreprocArena* arena) { if (state->unq_liter_germ.count > 0) { Chunk chu = { .qmode = UN_Q, .kind = LITERAL, .as.literal_str = inner(&state->unq_liter_germ) }; Chunk* chu_ptr = push_to_ChunkVec(&arena->chunks, chu); push_to_Word(&state->word_germ, chu_ptr); state->unq_liter_germ = make_String(256); } if (state->word_germ.count > 0) { Token tok = {.kind = TOK_WORD, .as.word = state->word_germ}; Token* tok_ptr = push_to_TokenVec(&arena->tokens, tok); push_to_TokenPtrVec(toks, tok_ptr); state->word_germ = make_Word(256); } } TokenPtrVec lex_level(RawLine* line, char until, PreprocArena* arena) { TokenPtrVec line_toks = make_TokenPtrVec(256); LexState state = { .word_germ = make_Word(256), .unq_liter_germ = make_String(256) }; while (line->str[line->pos] != until) { if (line->str[line->pos] == ' ') { close_word_and_lit_germs(&state, &line_toks, arena); while (line->str[line->pos] == ' ') line->pos++; continue; } switch (line->str[line->pos]) { case '\'': Chunk sqchunk = collect_single_quoted_section(line); Chunk* sqptr = push_to_ChunkVec(&arena->chunks, sqchunk); push_to_Word(&state.word_germ, sqptr); break; case '\"': lex_double_quoted_section(line, &state.word_germ, arena); break; case '|': line->pos++; close_word_and_lit_germs(&state, &line_toks, arena); Token* tok_ptr = push_to_TokenVec(&arena->tokens, (Token){.kind = TOK_SPECIAL, .as.spec = PIPE}); push_to_TokenPtrVec(&line_toks, tok_ptr); break; case '$': exit(1); /* line->pos++; MaybeChunk mchunk = lex_expansion_chunk(line, reg_arena, UN_Q); if (mchunk.some) { push_to_Word(&state.word_germ, mchunk.value); } else { push_to_String(&state.unq_liter_germ, '$'); } */ break; case '>': close_word_and_lit_germs(&state, &line_toks, arena); line->pos++; switch (line->str[line->pos]) { case '(': line->pos++; TokenPtrVec sub_regtoks = lex_level(line, ')', arena); line->pos++; Region subreg = {.mode = CMDS, .tokens = sub_regtoks}; Region* subreg_ptr = push_to_RegionVec(&arena->regions, subreg); Token tok = {.kind = TOK_PROCSUB_OUT, .as.procsub_region = subreg_ptr}; Token* tok_ptr = push_to_TokenVec(&arena->tokens, tok); push_to_TokenPtrVec(&line_toks, tok_ptr); break; case '>': line->pos++; Token* tok_ptr2 = push_to_TokenVec(&arena->tokens, (Token){.kind = TOK_SPECIAL, .as.spec = WRITE_REDIR_APPEND}); push_to_TokenPtrVec(&line_toks, tok_ptr2); break; default: Token* tok_ptr3 = push_to_TokenVec(&arena->tokens, (Token){.kind = TOK_SPECIAL, .as.spec = WRITE_REDIR_TRUNC}); push_to_TokenPtrVec(&line_toks, tok_ptr3); } break; case '<': close_word_and_lit_germs(&state, &line_toks, arena); line->pos++; if (line->str[line->pos] == '(') { line->pos++; TokenPtrVec sub_regtoks = lex_level(line, ')', arena); line->pos++; Region subreg = {.mode = CMDS, .tokens = sub_regtoks}; Region* reg_ptr = push_to_RegionVec(&arena->regions, subreg); Token reg_tok = {.kind = TOK_PROCSUB_IN, .as.procsub_region = reg_ptr}; Token* reg_tok_ptr = push_to_TokenVec(&arena->tokens, reg_tok); push_to_TokenPtrVec(&line_toks, reg_tok_ptr); } else { Token* redir_ptr = push_to_TokenVec(&arena->tokens, (Token){.kind = TOK_SPECIAL, .as.spec = READ_REDIR}); push_to_TokenPtrVec(&line_toks, redir_ptr); } break; default: push_to_String(&state.unq_liter_germ, line->str[line->pos++]); } } // TODO: handle until char _and_ EOL close_word_and_lit_germs(&state, &line_toks, arena); return line_toks; } TokenPtrVec lex(char* input, PreprocArena* arena) { RawLine line = {.str = input, .pos = 0}; TokenPtrVec toks = lex_level(&line, '\0', arena); Token* eol_ptr = push_to_TokenVec(&arena->tokens, (Token){.kind = TOK_SPECIAL, .as.spec = EOL}); push_to_TokenPtrVec(&toks, eol_ptr); return toks; }