Question

i need help making a scanner with nested (embedded) cases in a C program, i started doing it but i still can't get it to work. The other TODOs I think i got them working. i need help with TOKEN *scanner(). input.txt should be a Program Argument. The scanner function should emulate your state machine to tokenize inputs, character by character, to create and return a single token. It will do so by reading 1 new character in each iteration of a while loop. It must use these characters (along with the current state of the finite state machine) to determine what state transitions to take, and to determine when to assign value to the token being constructed. When a token is composed of multiple characters, you'll generally know it's "done" when you encounter a character that is not part of the token being composed (for instance, if your state machine is in the INTEGER_STATE and you encounter a letter, that letter is clearly not part of the integer, so it must be part of the next token.When this happens, you'll want to put the unused character back on the input stream so it can be reconsidered as the beginning of the following token. Check out the manual entry for ungetc.

input.txt

firstvar = 123;

secondVar = 21.;

var3 = 1.5;

repeat (10)

var3 = 2 * (firstvar + secondvar) / (firstvar + 2);

repeat (firstvar + 2 * secondvar)

print firstvar;

main.c

#include "scanner.h"

int main(int argc, char **argv)

{

freopen(argv[1], "r", stdin);

TOKEN *token = NULL;

printf("\n");

do

{

freeToken(&token);

token = scanner();

printToken(&token);

fflush(stdout);

}

while (token->type != EOF_TOKEN);

freeToken(&token);

printf("\n");

exit(EXIT_SUCCESS);

}

scanner.h

#ifndef __SCANNER_H

#define __SCANNER_H

#include <stdio.h>

#include <stdlib.h>

#include <string.h>

#include <stdbool.h>

typedef enum

{

NO_TOKEN_TYPE,

INVALID_TOKEN,

REPEAT_TOKEN,

PRINT_TOKEN,

IDENT_TOKEN,

INT_TOKEN,

FLOAT_TOKEN,

ASSIGNMENT_TOKEN,

LPAREN_TOKEN,

RPAREN_TOKEN,

ADD_OP_TOKEN,

MULT_OP_TOKEN,

SEMICOLON_TOKEN,

EOF_TOKEN

} TOKEN_TYPE;

typedef union

{

long integer;

double floating_point;

char *string;

char op;

} TOKEN_VALUE;

typedef struct token

{

TOKEN_TYPE type;

TOKEN_VALUE val;

} TOKEN;

typedef enum

{

START_STATE = 0,

ID_OR_KEYWORD_STATE,

INT_STATE,

FLOAT_STATE

} STATE;

TOKEN *scanner();

void freeToken(TOKEN **);

void printToken(TOKEN **);

#define BUF_SIZE 128

#define MAX_LINE_LENGTH 256

#endif

scanner.c

#include "scanner.h"

#include <stdio.h>

#include <stdlib.h>

#include <string.h>

#include <stdbool.h>

#define BUFFER_SIZE 32

//

// clean up the referenced token (if it isn't NULL).

//

void freeToken(TOKEN **token)

{

/*

* TODO

* free the referenced TOKEN *, and any contained data

* which requires freeing.

*

* Then, set the referenced TOKEN * to NULL

*/

if (*token == NULL)

return;

free((*token)->val.string);

free(*token);

*token = NULL;

}

bool updateKeywordOrId(TOKEN *token, char *str)

{

/*

* TODO

* Check if the collected token is a keyword by comparing its string

* value to the string values of the print and repeat keywords.

*

* If the token is a keyword, change its type to the corresponding

* TOKEN_TYPE and return true.

*

* If the token is not a keyword, change its type to the token type

* for identifiers, allocate space for its string value and copy the

* string value into the token, then return false.

*/

if(strcmp(str, "repeat")==0)

token->type=REPEAT_TOKEN;

else if(strcmp(str, "print") == 0)

token->type = PRINT_TOKEN;

else

return false;

return true;

}

void printToken (TOKEN **token)

{

/*

* TODO

* Print the referenced token in a readable format.

* Displayed information should include the TOKEN_TYPE,

* and also the token's value if applicable.

*/

printf("<%u %s>\n", (*token)->type, (*token)->val.string);

}

TOKEN *scanner()

{

// buffer to store a token's contained characters while it is being tokenized

size_t bufferSize = BUFFER_SIZE;

char *stringValue = calloc(sizeof(char), bufferSize);

int stringValueIndex = 0;

// allocate space for the new token

TOKEN *token = (TOKEN *) malloc(sizeof(TOKEN));

// initialize the token type to invalid

token->type = NO_TOKEN_TYPE;

// set state machine to starting state

STATE state = START_STATE;

char currentChar = '\0';

while (currentChar != EOF && token->type == NO_TOKEN_TYPE)

{

currentChar = (char) getchar();

/*

* TODO

* Given the current state of the state machine

* and the next character, update the state machine

* (and the string value of the token being built,

* if applicable).

*/

switch(token->type)

{

case 1:

switch(currentChar)

{

case '=':

token->type= ASSIGNMENT_TOKEN;

break;

case '(':

token->type= LPAREN_TOKEN;

break;

case ')':

token->type= RPAREN_TOKEN;

break;

case '0'...'9':

token->type= INT_TOKEN;

break;

case 'A'...'z':

token->type= IDENT_TOKEN;

break;

case '+':

case '-':

token->type= ADD_OP_TOKEN;

case '/':

case '*':

case '%':

token->type= MULT_OP_TOKEN;

case EOF:

token->type=EOF_TOKEN;

case ';':

token->type= SEMICOLON_TOKEN;

case '.':

token->type =FLOAT_TOKEN;

}

case INT_TOKEN:

switch (currentChar)

{

case '0'...'9':

break;

case '.':

break;

default:

ungetc(currentChar, (FILE *) "../input.txt"); // i think this is how it works.

return token;

}

case IDENT_TOKEN:

switch (currentChar)

{

case 'A'...'z':

break;

case '0'...'9':

break;

// print and repeat tokens should go here maybe

default:

ungetc(currentChar,(FILE *) "../input.txt");

return token;

}

}

// if the buffer is full, double its size

if (stringValueIndex >= bufferSize-1)

{

bufferSize *= 2;

char *temp = calloc(sizeof(char), bufferSize);

strcpy(temp, stringValue);

free(stringValue);

stringValue = temp;

}

}

free(stringValue);

return token;

}

Answer 1

Program Files

scanner.c

#include "scanner.h"

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdbool.h>

#define BUFFER_SIZE 32
#define PRINT_KEYWORD "print"
#define REPEAT_KEYWORD "repeat"

//
// clean up the referenced token (if it isn't NULL).
//
void freeToken(TOKEN **token)
{
/*
* TODO
* free the referenced TOKEN *, and any contained data
* which requires freeing.
*
* Then, set the referenced TOKEN * to NULL
*/
if(*token != NULL) {
switch ((*token)->type) {
case IDENT_TOKEN:
case PRINT_TOKEN:
case REPEAT_TOKEN:
free((*token)->val.string);
free(*token);
break;
default:
free(*token);
break;
}
*token = NULL;
}
}

bool updateKeywordOrId(TOKEN *token, char *str)
{
/*
* TODO
* Check if the collected token is a keyword by comparing its string
* value to the string values of the print and repeat keywords.
*
* If the token is a keyword, change its type to the corresponding
* TOKEN_TYPE and return true.
*
* If the token is not a keyword, change its type to the token type
* for identifiers, allocate space for its string value and copy the
* string value into the token, then return false.
*
*/
char * temp = NULL;
bool ret = true;
if(strcmp(PRINT_KEYWORD, str) == 0){
token->type = PRINT_TOKEN;
temp = calloc(sizeof(char), strlen(PRINT_KEYWORD));
strcpy(temp,PRINT_KEYWORD);

} else if (strcmp(REPEAT_KEYWORD, str) == 0){
token->type = REPEAT_TOKEN;
temp = calloc(sizeof(char), strlen(REPEAT_KEYWORD));
strcpy(temp,REPEAT_KEYWORD);

} else {
token->type = IDENT_TOKEN;
temp = calloc(sizeof(char), strlen(str));
strcpy(temp, str);
ret = false;
}
token->val.string = temp;

return ret;
}

void printToken (TOKEN **token)
{
/*
* TODO
* Print the referenced token in a readable format.
* Displayed information should include the TOKEN_TYPE,
* and also the token's value if applicable.
*
*/
switch((*token)->type){
case INT_TOKEN:
printf("<INT, %ld >\n", (*token)->val.integer);
break;
case IDENT_TOKEN:
printf("<IDENT, %s>\n", (*token)->val.string);
break;
case FLOAT_TOKEN:
printf("<FLOAT, %f>\n", (*token)->val.floating_point);
break;
case EOF_TOKEN:
printf("<EOF>\n");
break;
case INVALID_TOKEN:
printf("<INVALID>\n");
break;
case SEMICOLON_TOKEN:
printf("<SEMICOLON>\n");
break;
case ASSIGNMENT_TOKEN:
printf("<ASSIGN>\n");
break;
case RPAREN_TOKEN:
printf("<RPAREN>\n");
break;
case LPAREN_TOKEN:
printf("<LPAREN>\n");
break;
case MULT_OP_TOKEN:
printf("<MULT_OP,%c>\n",(*token)->val.op);
break;
case ADD_OP_TOKEN:
printf("<ADD_OP,%c>\n",(*token)->val.op);
break;
case PRINT_TOKEN:
case REPEAT_TOKEN:
printf("<KEYWORD, %s>\n",(*token)->val.string);
break;
default:
break;

}

}

TOKEN *scanner()
{
// buffer to store a token's contained characters while it is being tokenized
size_t bufferSize = BUFFER_SIZE;
char *stringValue = calloc(sizeof(char), bufferSize);
int stringValueIndex = 0;

// allocate space for the new token
TOKEN *token = (TOKEN *) malloc(sizeof(TOKEN));

// initialize the token type to invalid
token->type = NO_TOKEN_TYPE;

// set state machine to starting state
STATE state = START_STATE;

char currentChar = '\0';

while (currentChar != EOF && token->type == NO_TOKEN_TYPE) {
currentChar = (char) getchar();
/*
* TODO
* Given the current state of the state machine
* and the next character, update the state machine
* (and the string value of the token being built,
* if applicable).
*/

switch (state) {
case START_STATE:
switch (currentChar) {
case '+':
case '-':
token->type = ADD_OP_TOKEN;
token->val.op = currentChar;
break;
case '*':
case '/':
token->type = MULT_OP_TOKEN;
token->val.op = currentChar;
break;
case '(':
token->type = LPAREN_TOKEN;
break;
case ')':
token->type = RPAREN_TOKEN;
break;
case '=':
token->type = ASSIGNMENT_TOKEN;
break;
case ';':
token->type = SEMICOLON_TOKEN;
break;
case '0'...'9':
state = INT_STATE;
stringValue[stringValueIndex] = currentChar;
stringValueIndex++;
break;
case 'A'...'z':
state = ID_OR_KEYWORD_STATE;
stringValue[stringValueIndex] = currentChar;
stringValueIndex++;
break;
case EOF :
token->type = EOF_TOKEN;
break;
case ' ':
case '\t':
case '\n':
state = START_STATE;
break;
default:
token->type = INVALID_TOKEN;
break;
}
break;

case ID_OR_KEYWORD_STATE:
switch (currentChar) {
case '0'...'9':
state = ID_OR_KEYWORD_STATE;
stringValue[stringValueIndex] = currentChar;
stringValueIndex++;
break;
case 'A'...'z':
state = ID_OR_KEYWORD_STATE;
stringValue[stringValueIndex] = currentChar;
stringValueIndex++;
break;
default:
ungetc(currentChar, stdin);
updateKeywordOrId(token, stringValue);
break;
}
break;
case INT_STATE:
switch (currentChar) {
case '0'...'9':
state = INT_STATE;
stringValue[stringValueIndex] = currentChar;
stringValueIndex++;
break;
case '.':
state = FLOAT_STATE;
stringValue[stringValueIndex] = currentChar;
stringValueIndex++;
break;
default:
ungetc(currentChar, stdin);
token->val.integer = atoi(stringValue);
token->type = INT_TOKEN;
break;
}
break;
case FLOAT_STATE:
switch (currentChar) {
case '0'...'9':
state = FLOAT_STATE;
stringValue[stringValueIndex] = currentChar;
stringValueIndex++;
break;
default:
ungetc(currentChar, stdin);
token->val.floating_point = atof(stringValue);
token->type = FLOAT_TOKEN;
break;
}
break;
}

// if the buffer is full, double its size
if (stringValueIndex >= bufferSize - 1) {
bufferSize *= 2;
char *temp = calloc(sizeof(char), bufferSize);
strcpy(temp, stringValue);
free(stringValue);
stringValue = temp;
}
}

free(stringValue);
return token;
}

scanner.h

#ifndef __SCANNER_H
#define __SCANNER_H

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdbool.h>

typedef enum
{
NO_TOKEN_TYPE,
INVALID_TOKEN,
REPEAT_TOKEN,
PRINT_TOKEN,
IDENT_TOKEN,
INT_TOKEN,
FLOAT_TOKEN,
ASSIGNMENT_TOKEN,
LPAREN_TOKEN,
RPAREN_TOKEN,
ADD_OP_TOKEN,
MULT_OP_TOKEN,
SEMICOLON_TOKEN,
EOF_TOKEN
} TOKEN_TYPE;

typedef union
{
long integer;
double floating_point;
char *string;
char op;
} TOKEN_VALUE;

typedef struct token
{
TOKEN_TYPE type;
TOKEN_VALUE val;
} TOKEN;

typedef enum
{
START_STATE = 0,
ID_OR_KEYWORD_STATE,
INT_STATE,
FLOAT_STATE
} STATE;

TOKEN *scanner();

void freeToken(TOKEN **);

void printToken(TOKEN **);

#define BUF_SIZE 128
#define MAX_LINE_LENGTH 256

#endif

input.txt

firstvar = 123;

secondVar = 21.;

var3 = 1.5;

repeat (10)
var3 = 2 * (firstvar + secondvar) / (firstvar + 2);

repeat (firstvar + 2 * secondvar)
print firstvar;

main.c

// main.c (driver for a scanner test)

#include "scanner.h"

int main(int argc, char **argv)
{
freopen(argv[1], "r", stdin);

TOKEN *token = NULL;
printf("\n");

do
{
freeToken(&token);
token = scanner();
printToken(&token);
fflush(stdout);
}
while (token->type != EOF_TOKEN);

freeToken(&token);

printf("\n");
exit(EXIT_SUCCESS);
}

output

 clang main.c scanner.c scanner.h
 ./a.out input.txt

<IDENT, firstvar>
<ASSIGN>
<INT, 123 >
<SEMICOLON>
<INVALID>
<INVALID>
<IDENT, secondVar>
<ASSIGN>
<FLOAT, 21.000000>
<SEMICOLON>
<INVALID>
<INVALID>
<IDENT, var3>
<ASSIGN>
<FLOAT, 1.500000>
<SEMICOLON>
<INVALID>
<INVALID>
<KEYWORD, repeat>
<LPAREN>
<INT, 10 >
<RPAREN>
<INVALID>
<IDENT, var3>
<ASSIGN>
<INT, 2 >
<MULT_OP,*>
<LPAREN>
<IDENT, firstvar>
<ADD_OP,+>
<IDENT, secondvar>
<RPAREN>
<MULT_OP,/>
<LPAREN>
<IDENT, firstvar>
<ADD_OP,+>
<INT, 2 >
<RPAREN>
<SEMICOLON>
<INVALID>
<INVALID>
<KEYWORD, repeat>
<LPAREN>
<IDENT, firstvar>
<ADD_OP,+>
<INT, 2 >
<MULT_OP,*>
<IDENT, secondvar>
<RPAREN>
<INVALID>
<KEYWORD, print>
<IDENT, firstvar>
<SEMICOLON>
<EOF>

Hope this helps!

Please let me know if any changes needed.

Thank you!