#include <stdio.h> #include <string.h> #include "char16.h" extern int offset; extern int token_length; extern char token[]; /*extern char *yytext;*/ extern int print_offset; extern int print_token_length; void maca_tokenizer_segment(char *separator, char *text_matched){ if(token_length != 0){ printf("%s", token); if(print_offset) printf("\t%d", offset); if(print_token_length) printf("\t%d", utf8_strlen(token)); printf("\n"); } offset += utf8_strlen(token); token_length = 0; token[0] = 0; if(strlen(separator) != 0){ printf("%s", separator); if(print_offset) printf("\t%d", offset); if(print_token_length) printf("\t%d", (int) utf8_strlen(separator)); printf("\n"); } offset += utf8_strlen(text_matched); } void maca_tokenizer_add_char_to_token(char c) { token[token_length] = c; token_length++; token[token_length] = 0; }