ref: 5095f44178e0cbf98419f1a4a22b5d0581813877
dir: /testsuite/token_test_main.c/
/*************************************************************************/ /* */ /* Language Technologies Institute */ /* Carnegie Mellon University */ /* Copyright (c) 1999 */ /* All Rights Reserved. */ /* */ /* Permission is hereby granted, free of charge, to use and distribute */ /* this software and its documentation without restriction, including */ /* without limitation the rights to use, copy, modify, merge, publish, */ /* distribute, sublicense, and/or sell copies of this work, and to */ /* permit persons to whom this work is furnished to do so, subject to */ /* the following conditions: */ /* 1. The code must retain the above copyright notice, this list of */ /* conditions and the following disclaimer. */ /* 2. Any modifications must be clearly marked as such. */ /* 3. Original authors' names are not deleted. */ /* 4. The authors' names are not used to endorse or promote products */ /* derived from this software without specific prior written */ /* permission. */ /* */ /* CARNEGIE MELLON UNIVERSITY AND THE CONTRIBUTORS TO THIS WORK */ /* DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING */ /* ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT */ /* SHALL CARNEGIE MELLON UNIVERSITY NOR THE CONTRIBUTORS BE LIABLE */ /* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES */ /* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN */ /* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, */ /* ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF */ /* THIS SOFTWARE. */ /* */ /*************************************************************************/ /* Author: Alan W Black (awb@cs.cmu.edu) */ /* Date: August 1999 */ /*************************************************************************/ /* */ /* test for tokenizer of strings and files */ /* */ /*************************************************************************/ #include <stdio.h> #include "cst_tokenstream.h" int main(int argc, char **argv) { /* need that cool argument parser */ cst_tokenstream *fd; const char *token; fd = ts_open("data.one", cst_ts_default_whitespacesymbols, cst_ts_default_singlecharsymbols, cst_ts_default_prepunctuationsymbols, cst_ts_default_postpunctuationsymbols); while (!ts_eof(fd)) { token = ts_get(fd); printf("ws >%s<\n",fd->whitespace); printf("pp >%s<\n",fd->prepunctuation); printf("tk >%s<\n",token); printf("pp >%s<\n",fd->postpunctuation); printf("fp >%d<\n",fd->file_pos); printf("ln >%d<\n",fd->line_number); printf("\n"); } ts_close(fd); return 0; }