maca_tokenizer_functions_for_lex.c

#include <stdio.h>
#include <string.h>
#include "char16.h"

extern int offset;
extern int token_length;
extern char token[];
/*extern char *yytext;*/
extern int print_offset;
extern int print_token_length;

void maca_tokenizer_segment(char *separator, char *text_matched){
  if(token_length != 0){
    printf("%s", token);
    if(print_offset)
      printf("\t%d", offset);
    if(print_token_length)
      printf("\t%d", utf8_strlen(token));
    printf("\n");
  }
  
  offset += utf8_strlen(token);
  token_length = 0;
  token[0] = 0;
  
  if(strlen(separator) != 0){
    printf("%s", separator);
    if(print_offset)
      printf("\t%d", offset);
    if(print_token_length)
      printf("\t%d", (int) utf8_strlen(separator));
    printf("\n");
  }
  offset += utf8_strlen(text_matched); 
}


void maca_tokenizer_add_char_to_token(char c)
{
  token[token_length] = c;
  token_length++;
  token[token_length] = 0;
}