Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Fix lx token identification #509

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 18 commits into from
Aug 20, 2025
Merged

Fix lx token identification #509

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
825303f
Remove dependence on internal.h in print/ir.h and lx/print/c.c.
silentbicycle Jul 3, 2025
cb42d58
Makefile: Check '*res*' not 'res*' for tests.
silentbicycle Jul 8, 2025
7824e6a
Fix lx by rewriting much of its -l c codegen.
silentbicycle Jul 3, 2025
26e2b13
test/lxpos: Update test data (EOF position info).
silentbicycle Jul 22, 2025
862a68c
Re-enable lxpos tests.
silentbicycle Jul 22, 2025
affca78
Use $LX_BIN instead of $LX in lxpos makefile.
silentbicycle Jul 23, 2025
a8f0c59
lx: Make -l dump's output call lx.free() when using dynamic buffer.
silentbicycle Jul 23, 2025
552aa01
lx: Use prefix.tok, not "TOK_".
silentbicycle Jul 29, 2025
beebd1b
lx: return TOK_ERROR if reaching the end of a zone function.
silentbicycle Jul 29, 2025
ae53e94
lx: Handle unexpected EOF in pattern pairs. Add tests, update out6.dump.
silentbicycle Jul 29, 2025
ea9c90b
lx: Rewrite logic to make the four cases explicit, fix dead code.
silentbicycle Jul 30, 2025
17d415d
lx: Only gen fixedpop / dynpop & calls to them when buffer mode is set.
silentbicycle Jul 31, 2025
7ed18b9
lx: Suppress warning for possibly unused function.
silentbicycle Jul 31, 2025
1e55db8
lx: Ensure prefix.api & prefix.lx are used in the generated code.
silentbicycle Jul 31, 2025
4a5ca84
Replace FSM_ADVANCE_HOOK macro with optional hooks->advance callback.
silentbicycle Aug 4, 2025
f25e8b7
The advance hook should also be called for FSM_IO_STR.
silentbicycle Aug 4, 2025
051aaf0
Move setting `has_consumed_input` flag into lx's advance hook.
silentbicycle Aug 4, 2025
08fd72c
lx: Avoid useless call to pop and some other 'unused' warnings.
silentbicycle Aug 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,8 +124,7 @@ SUBDIR += tests/fsm
SUBDIR += tests/glob
SUBDIR += tests/like
SUBDIR += tests/literal
# FIXME: commenting this out for now due to Makefile error
#SUBDIR += tests/lxpos
SUBDIR += tests/lxpos
SUBDIR += tests/minimise
SUBDIR += tests/native
SUBDIR += tests/pcre
Expand Down Expand Up @@ -190,6 +189,6 @@ STAGE_BUILD := ${STAGE_BUILD:Nbin/cvtpcre}

.if make(test)
.END::
grep FAIL ${BUILD}/tests/*/res*; [ $$? -ne 0 ]
grep FAIL ${BUILD}/tests/*/*res*; [ $$? -ne 0 ]
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This may have been causing test failures to go unnoticed before.

.endif

7 changes: 7 additions & 0 deletions include/fsm/print.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,15 @@ struct fsm_hooks {
void *lang_opaque, void *hook_opaque);

int (*reject)(FILE *, const struct fsm_options *opt,
const struct fsm_state_metadata *state_metadata,
void *lang_opaque, void *hook_opaque);

/* If non-NULL, this will be called to generate code
* in scope immediately after advancing to the
* next character of input. */
int (*advance)(FILE *, const struct fsm_options *opt,
const char *cur_char_var, void *hook_opaque);

int (*comment)(FILE *, const struct fsm_options *opt,
const struct fsm_state_metadata *state_metadata,
void *hook_opaque);
Expand Down
3 changes: 2 additions & 1 deletion src/libfsm/print.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ int
print_hook_reject(FILE *f,
const struct fsm_options *opt,
const struct fsm_hooks *hooks,
const struct fsm_state_metadata *state_metadata,
int (*default_reject)(FILE *f, const struct fsm_options *opt,
void *lang_opaque, void *hook_opaque),
void *lang_opaque)
Expand All @@ -124,7 +125,7 @@ print_hook_reject(FILE *f,
}

if (hooks->reject != NULL) {
return hooks->reject(f, opt,
return hooks->reject(f, opt, state_metadata,
lang_opaque, hooks->hook_opaque);
} else if (default_reject != NULL) {
return default_reject(f, opt,
Expand Down
1 change: 1 addition & 0 deletions src/libfsm/print.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ int
print_hook_reject(FILE *f,
const struct fsm_options *opt,
const struct fsm_hooks *hooks,
const struct fsm_state_metadata *state_metadata,
int (*default_reject)(FILE *f, const struct fsm_options *opt,
void *lang_opaque, void *hook_opaque),
void *lang_opaque);
Expand Down
2 changes: 1 addition & 1 deletion src/libfsm/print/awk.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op,
{
switch (end_bits) {
case VM_END_FAIL:
return print_hook_reject(f, opt, hooks, default_reject, NULL);
return print_hook_reject(f, opt, hooks, NULL, default_reject, NULL);

case VM_END_SUCC:;
struct fsm_state_metadata state_metadata = {
Expand Down
24 changes: 18 additions & 6 deletions src/libfsm/print/c.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ print_groups(FILE *f, const struct fsm_options *opt,
}

static int
print_case(FILE *f, const struct ir *ir,
print_case(FILE *f, const struct ir *ir, fsm_state_t state_id,
const struct fsm_options *opt,
const struct fsm_hooks *hooks,
const char *cp,
Expand All @@ -222,10 +222,16 @@ print_case(FILE *f, const struct ir *ir,
assert(f != NULL);
assert(cs != NULL);

assert(state_id < ir->n);
const struct fsm_state_metadata state_metadata = {
.end_ids = ir->states[state_id].endids.ids,
.end_id_count = ir->states[state_id].endids.count,
};

Copy link
Owner

@katef katef Aug 2, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why not pass whatever type &ir->states[state_id].endids is?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

struct ir_state_endids is defined in an internal ir.h header, and an upcoming PR is going to add another pair of fields to the state metadata struct, for eager output IDs and their count.

switch (cs->strategy) {
case IR_NONE:
fprintf(f, "\t\t\t");
if (-1 == print_hook_reject(f, opt, hooks, default_reject, NULL)) {
if (-1 == print_hook_reject(f, opt, hooks, &state_metadata, default_reject, NULL)) {
return -1;
}
fprintf(f, "\n");
Expand Down Expand Up @@ -254,7 +260,7 @@ print_case(FILE *f, const struct ir *ir,
print_groups(f, opt, ir_indexof(ir, cs), cs->u.partial.groups, cs->u.partial.n);

fprintf(f, "\t\t\tdefault: ");
if (-1 == print_hook_reject(f, opt, hooks, default_reject, NULL)) {
if (-1 == print_hook_reject(f, opt, hooks, &state_metadata, default_reject, NULL)) {
return -1;
}
fprintf(f, "\n");
Expand Down Expand Up @@ -285,7 +291,7 @@ print_case(FILE *f, const struct ir *ir,

print_ranges(f, opt, cs->u.error.error.ranges, cs->u.error.error.n);
fprintf(f, " ");
if (-1 == print_hook_reject(f, opt, hooks, default_reject, NULL)) {
if (-1 == print_hook_reject(f, opt, hooks, &state_metadata, default_reject, NULL)) {
return -1;
}
fprintf(f, "\n");
Expand Down Expand Up @@ -398,7 +404,7 @@ print_endstates(FILE *f,

/* unexpected EOT */
fprintf(f, "\tdefault: ");
if (-1 == print_hook_reject(f, opt, hooks, default_reject, NULL)) {
if (-1 == print_hook_reject(f, opt, hooks, NULL, default_reject, NULL)) {
return -1;
}
fprintf(f, "\n");
Expand Down Expand Up @@ -435,7 +441,7 @@ fsm_print_cfrag(FILE *f, const struct ir *ir,
}
fprintf(f, "\n");

if (-1 == print_case(f, ir, opt, hooks, cp, &ir->states[i])) {
if (-1 == print_case(f, ir, i, opt, hooks, cp, &ir->states[i])) {
return -1;
}

Expand Down Expand Up @@ -500,6 +506,12 @@ fsm_print_c_body(FILE *f, const struct ir *ir,
break;
}

if (hooks->advance != NULL) {
if (-1 == hooks->advance(f, opt, cp, hooks->hook_opaque)) {
return -1;
}
}

if (-1 == fsm_print_cfrag(f, ir, opt, hooks, cp)) {
return -1;
}
Expand Down
2 changes: 1 addition & 1 deletion src/libfsm/print/dot.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ print_dotfrag(FILE *f,
if (!opt->anonymous_states) {
fprintf(f, "\t%sS%-2u [ ", prefix, s);

if (-1 == print_hook_reject(f, opt, hooks, default_reject, &s)) {
if (-1 == print_hook_reject(f, opt, hooks, NULL, default_reject, &s)) {
return -1;
}

Expand Down
2 changes: 1 addition & 1 deletion src/libfsm/print/fsm.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ print_state(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hook
assert(opt != NULL);

if (!fsm_isend(fsm, s)) {
if (-1 == print_hook_reject(f, opt, hooks, NULL, NULL)) {
if (-1 == print_hook_reject(f, opt, hooks, NULL, NULL, NULL)) {
return -1;
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/libfsm/print/go.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op,

switch (end_bits) {
case VM_END_FAIL:
return print_hook_reject(f, opt, hooks, default_reject, NULL);
return print_hook_reject(f, opt, hooks, NULL, default_reject, NULL);

case VM_END_SUCC:
assert(op->ret >= retlist->a);
Expand Down
6 changes: 1 addition & 5 deletions src/libfsm/print/ir.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,11 +91,7 @@ struct ir_state {
} error;

struct {
/* Note: This is allocated separately, to avoid
* making the union significantly larger. */
struct ir_state_table {
unsigned to[FSM_SIGMA_COUNT];
} *table;
int not_yet_implemented;
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nothing uses this yet, but referencing FSM_SIGMA_COUNT here made the ir.h header depend on internal.h.

} table;
} u;
};
Expand Down
2 changes: 1 addition & 1 deletion src/libfsm/print/llvm.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -713,7 +713,7 @@ fsm_print_llvm(FILE *f,
fprintf(f, "\n");
}
fprintf(f, "\t ");
if (-1 == print_hook_reject(f, opt, hooks, default_reject, NULL)) {
if (-1 == print_hook_reject(f, opt, hooks, NULL, default_reject, NULL)) {
return -1;
}
fprintf(f, "\n");
Expand Down
2 changes: 1 addition & 1 deletion src/libfsm/print/rust.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op,

switch (end_bits) {
case VM_END_FAIL:
return print_hook_reject(f, opt, hooks, default_reject, NULL);
return print_hook_reject(f, opt, hooks, NULL, default_reject, NULL);

case VM_END_SUCC:
assert(op->ret >= retlist->a);
Expand Down
2 changes: 1 addition & 1 deletion src/libfsm/print/sh.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op,
{
switch (end_bits) {
case VM_END_FAIL:
return print_hook_reject(f, opt, hooks, default_reject, NULL);
return print_hook_reject(f, opt, hooks, NULL, default_reject, NULL);

case VM_END_SUCC:;
const struct fsm_state_metadata state_metadata = {
Expand Down
2 changes: 1 addition & 1 deletion src/libfsm/print/vmasm.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op,

switch (end_bits) {
case VM_END_FAIL:
if (-1 == print_hook_reject(f, opt, hooks, NULL, NULL)) {
if (-1 == print_hook_reject(f, opt, hooks, NULL, NULL, NULL)) {
return -1;
}
break;
Expand Down
2 changes: 1 addition & 1 deletion src/libfsm/print/vmc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op,
{
switch (end_bits) {
case VM_END_FAIL:
return print_hook_reject(f, opt, hooks, default_reject, NULL);
return print_hook_reject(f, opt, hooks, NULL, default_reject, NULL);

case VM_END_SUCC:;
struct fsm_state_metadata state_metadata = {
Expand Down
2 changes: 1 addition & 1 deletion src/libfsm/print/vmdot.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ print_end(FILE *f,
{
switch (end_bits) {
case VM_END_FAIL:
return print_hook_reject(f, opt, hooks, default_reject, NULL);
return print_hook_reject(f, opt, hooks, NULL, default_reject, NULL);

case VM_END_SUCC:;
struct fsm_state_metadata state_metadata = {
Expand Down
2 changes: 1 addition & 1 deletion src/libfsm/print/vmops.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ print_end(FILE *f, const struct dfavm_op_ir *op,

switch (end_bits) {
case VM_END_FAIL:
if (-1 == print_hook_reject(f, opt, hooks, default_reject, NULL)) {
if (-1 == print_hook_reject(f, opt, hooks, NULL, default_reject, NULL)) {
return -1;
}
break;
Expand Down
4 changes: 4 additions & 0 deletions src/lx/main.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -695,6 +695,10 @@ main(int argc, char *argv[])
opt.comments = 0;
}

if (lang == LX_PRINT_C) {
opt.fragment = 1;
}

{
if (print_progress) {
fprintf(stderr, "-- parsing:");
Expand Down
Loading