feat: implement lexer+parser for example program

.gitignore

@@ -1,6 +1,7 @@
 /*
 
 # Dirs
+!/exl
 !/include
 !/src
 

CMakeLists.txt

@@ -0,0 +1,18 @@
+cmake_minimum_required(VERSION 4.3)
+project(fz-lang
+	LANGUAGES C)
+
+include_directories(include)
+
+# Compiler
+add_executable(fzc
+	src/compiler/fzl_token.c
+	src/compiler/fzl_reader.c
+	src/compiler/fzl_parser.c
+	src/compiler/fzl_debug.c
+	src/compiler/main.c)
+
+# VM
+# add_executable(fzvm
+# 	src/vm/main.c)
+

Makefile

@@ -0,0 +1,8 @@
+.PHONY: configure build
+
+configure:
+	cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug
+
+build: configure
+	cmake --build build
+

include/compiler/fzl_debug.h

@@ -0,0 +1,10 @@
+#ifndef FZL_DEBUG_H
+#define FZL_DEBUG_H
+
+#include <compiler/fzl_parser.h>
+
+// Print AST
+void fzl_debug_print_ast(fzl_node_t *root);
+
+#endif//FZL_DEBUG_H
+

include/compiler/fzl_parser.h

@@ -0,0 +1,58 @@
+#ifndef FZL_PARSER_H
+#define FZL_PARSER_H
+
+#include <stddef.h>
+#include <compiler/fzl_token.h>
+#include <compiler/fzl_reader.h>
+
+// AST node type
+typedef enum {
+	FZL_NODETYPE_BLOCK,
+	FZL_NODETYPE_CALL,
+	FZL_NODETYPE_LITERAL,
+} fzl_nodetype_t;
+
+// AST node descriptor
+typedef struct fzl_node {
+	fzl_nodetype_t type;
+	fzl_token_t *token;
+	struct fzl_node *parent;
+	struct fzl_node *next;
+
+	union {
+		struct {
+			struct fzl_node **entries;
+			size_t entry_count;
+		} list;
+	};
+} fzl_node_t;
+
+// Token sequence parser descriptor
+typedef struct {
+	fzl_reader_ll_t *first;
+	size_t token_count;
+
+	fzl_reader_ll_t *cursor;
+	fzl_node_t *root;
+} fzl_parser_t;
+
+// Allocate parser
+fzl_parser_t *fzl_mkparser(fzl_reader_ll_t *first);
+
+// Allocate new node
+fzl_node_t *fzl_mknode(fzl_nodetype_t type, fzl_token_t *tok);
+
+// Parse tokens
+int fzl_parse(fzl_parser_t *parser);
+
+// Get last parser error
+char *fzl_parser_geterror();
+
+// Free node
+void fzl_freenode(fzl_node_t* node);
+
+// Free parser
+void fzl_freeparser(fzl_parser_t *parser);
+
+#endif//FZL_PARSER_H
+

include/compiler/fzl_reader.h

@@ -0,0 +1,38 @@
+#ifndef FZL_READER_H
+#define FZL_READER_H
+
+#include <stdint.h>
+#include <stddef.h>
+#include <stdbool.h>
+#include <compiler/fzl_token.h>
+
+// Tokens linked list descriptor
+typedef struct fzl_reader_ll {
+	fzl_token_t *token;
+	struct fzl_reader_ll *next;
+} fzl_reader_ll_t;
+
+// Source reader description
+typedef struct {
+	fzl_reader_ll_t *first;
+	fzl_reader_ll_t *last;
+
+	char *source;
+	size_t position;
+	bool at_end;
+} fzl_reader_t;
+
+// Allocate new reader
+fzl_reader_t *fzl_mkreader(char *source);
+
+// Read source file
+int fzl_read(fzl_reader_t *reader);
+
+// Get last error from reader
+char *fzl_reader_geterror();
+
+// Free reader
+void fzl_freereader(fzl_reader_t *reader);
+
+#endif//FZL_READER_H
+

include/compiler/fzl_token.h

@@ -0,0 +1,36 @@
+#ifndef FZL_TOKEN_H
+#define FZL_TOKEN_H
+
+#include <stdint.h>
+#include <stddef.h>
+#include <stdbool.h>
+
+// Token type
+typedef enum {
+	FZL_TOKTYPE_NAME,
+	FZL_TOKTYPE_LPAIR,
+	FZL_TOKTYPE_RPAIR,
+	FZL_TOKTYPE_COMMA,
+	FZL_TOKTYPE_STR_LITERAL
+} fzl_toktype_t;
+
+// Token descriptor
+typedef struct {
+	fzl_toktype_t type;
+	uint16_t i0;
+	uint16_t i1;
+	bool store_value;
+	void *value;
+	uint16_t value_sz;
+} fzl_token_t;
+
+// Allocate new token
+fzl_token_t *fzl_mktoken(
+	const fzl_toktype_t type, uint16_t from, uint16_t to,
+	void *value, size_t value_sz);
+
+// Free token and its contents
+void fzl_freetoken(fzl_token_t *token);
+
+#endif//FZL_TOKEN_H
+

src/compiler/fzl_debug.c

@@ -0,0 +1,82 @@
+#include <compiler/fzl_debug.h>
+#include <stdio.h>
+#include <stddef.h>
+#include <string.h>
+
+size_t print_level(char *buf, int level)
+{
+	int lvl = 0;
+	while (lvl < level)
+	{
+		buf[lvl] = '-';
+		lvl++;
+	}
+
+	buf[lvl] = '>';
+	return strlen(buf);
+}
+
+void print_node(fzl_node_t *node, int level)
+{
+	char levelbuf[32];
+	char printbuf[256];
+	memset(levelbuf, 0, sizeof(levelbuf));
+	memset(printbuf, 0, sizeof(printbuf));
+
+	if (!node) return;
+	
+	size_t lvllen = print_level(levelbuf, level);
+	strcpy(printbuf, levelbuf);
+
+	switch (node->type)
+	{
+		case FZL_NODETYPE_CALL:
+			if (node->list.entry_count > 0)
+			{
+				printf("%s Call fn '%s' with args:\n", levelbuf, (char*)node->token->value);
+				for (size_t i = 0; i < node->list.entry_count; i++)
+					print_node(node->list.entries[i], level + 1);
+			}
+			else printf("%s Call fn '%s' w/o args\n", levelbuf, (char*)node->token->value);
+			break;
+
+		case FZL_NODETYPE_LITERAL:
+			printf("%s Literal '%s'\n",
+						 levelbuf,
+						 (char*)node->token->value);
+			break;
+
+		case FZL_NODETYPE_BLOCK:
+			printf("%s Block: ", levelbuf);
+
+			if (node->list.entry_count < 1)
+			{
+				printf("<empty>\n");
+				break;
+			}
+			
+			printf("\n");
+			for (size_t i = 0; i < node->list.entry_count; i++)
+				print_node(node->list.entries[i], level + 1);
+
+			break;
+
+		default:
+			printf("%s Secure node\n", levelbuf);
+	}
+}
+
+void fzl_debug_print_ast(fzl_node_t *root)
+{
+	printf("> AST tree:\n");
+
+	fzl_node_t *curr = root;
+	while (curr != NULL)
+	{
+		print_node(curr, 2);
+		curr = curr->next;
+	}
+
+	printf("-> FINISH\n");
+}
+

src/compiler/fzl_parser.c

@@ -0,0 +1,214 @@
+#include <compiler/fzl_parser.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+char last_err[256];
+
+fzl_parser_t *fzl_mkparser(fzl_reader_ll_t *first)
+{
+	if (!first)
+	{
+		snprintf(last_err, sizeof(last_err), "token is null");
+		return NULL;
+	}
+
+	fzl_parser_t *parser = malloc(sizeof(fzl_parser_t));
+	if (!parser) return NULL;
+
+	parser->root = NULL;
+	parser->cursor = first->next;
+	if (!parser->cursor) return NULL;
+
+	return parser;
+}
+
+// --- Parse zone ---
+
+// Make a new node
+fzl_node_t *fzl_mknode(fzl_nodetype_t type, fzl_token_t *token)
+{
+	fzl_node_t *node = malloc(sizeof(fzl_node_t));
+
+	if (!node) return NULL;
+	node->type = type;
+	node->token = token;
+	node->parent = NULL;
+	node->next = NULL;
+
+	return node;
+}
+
+// Get token at cursor
+static fzl_token_t *peek(fzl_parser_t *parser)
+{
+	if (!parser->cursor) return NULL;
+	return parser->cursor->token;
+}
+
+// Get token at (cursor + 1)
+static fzl_token_t *next(fzl_parser_t *parser)
+{
+	if (!parser->cursor) return NULL;
+	
+	fzl_reader_ll_t *next = parser->cursor->next;
+	if (!next) return NULL;
+
+	return next->token;
+}
+
+// Consume token at cursor and move to the next
+static fzl_token_t *consume(fzl_parser_t *parser)
+{
+	if (!parser->cursor) return NULL;
+	fzl_token_t *tok = parser->cursor->token;
+	parser->cursor = parser->cursor->next;
+	return tok;
+}
+
+// Parse primary
+fzl_node_t *parse_primary(fzl_parser_t *parser)
+{
+	if (peek(parser) && peek(parser)->type == FZL_TOKTYPE_STR_LITERAL)
+	{
+		fzl_token_t *tok = consume(parser);
+		return fzl_mknode(FZL_NODETYPE_LITERAL, tok);
+	}
+
+	snprintf(last_err, sizeof(last_err),
+					 "failed to parse primary at %d", peek(parser)->i0);
+	return NULL;
+}
+
+fzl_node_t *parse_expression(fzl_parser_t *parser, int precedence);
+
+// Parse call expression
+fzl_node_t *parse_call(fzl_parser_t *parser)
+{
+	fzl_token_t *tok = consume(parser); // <NAME>
+	consume(parser); // (
+	
+	fzl_node_t *node = fzl_mknode(FZL_NODETYPE_CALL, tok);
+	node->list.entry_count = 0;
+
+	size_t capacity = 4;
+	node->list.entries = malloc(capacity * sizeof(fzl_node_t*));
+	node->list.entry_count = 0;
+
+	if (peek(parser) && peek(parser)->type == FZL_TOKTYPE_RPAIR)
+	{
+		consume(parser);
+		return node;
+	}
+
+	while (peek(parser) && peek(parser)->type != FZL_TOKTYPE_RPAIR)
+	{
+		fzl_node_t *arg = parse_expression(parser, 0);
+		if (!arg) return NULL;
+
+		if (node->list.entry_count == capacity)
+		{
+			capacity *= 2;
+			node->list.entries = realloc(node->list.entries, capacity * sizeof(fzl_node_t*));
+		}
+
+		node->list.entries[node->list.entry_count++] = arg;
+
+		if (peek(parser) && peek(parser)->type == FZL_TOKTYPE_COMMA)
+			consume(parser);
+		else if (peek(parser) && peek(parser)->type != FZL_TOKTYPE_RPAIR)
+		{
+			snprintf(last_err, sizeof(last_err), "expected ',' or ')' after arg list");
+			return NULL;
+		}
+	}
+
+	consume(parser);
+	return node;
+}
+
+// Expression parsing
+fzl_node_t *parse_expression(fzl_parser_t *parser, int precedence)
+{
+	if (peek(parser) && peek(parser)->type == FZL_TOKTYPE_NAME)
+	{
+		if (next(parser) && next(parser)->type == FZL_TOKTYPE_LPAIR)
+			return parse_call(parser);
+	}
+	else if (peek(parser) && peek(parser)->type == FZL_TOKTYPE_STR_LITERAL)
+		return parse_primary(parser);
+
+	snprintf(last_err, sizeof(last_err),
+					 "failed to parse expression at %d",
+					 peek(parser)->i0);
+	return NULL;
+}
+
+// Statement parsing
+fzl_node_t *parse_statement(fzl_parser_t *parser)
+{
+	return parse_expression(parser, 0);
+}
+
+// Internal parsing entry point
+fzl_node_t *parse_program(fzl_parser_t *parser)
+{
+	fzl_node_t *first = NULL;
+	fzl_node_t *last = NULL;
+	fzl_node_t *node = NULL;
+	while (peek(parser) != NULL)
+	{
+		node = parse_statement(parser);
+		if (!node) return NULL;
+
+		if (!last)
+		{
+			first = node;
+			last = node;
+		}
+		else
+		{
+			last->next = node;
+			last = node;
+		}
+	}
+
+	if (!first)
+	{
+		snprintf(last_err, sizeof(last_err), "empty ;(");
+		return NULL;
+	}
+
+	return first;
+}
+
+
+// Parse tokens and build AST
+int fzl_parse(fzl_parser_t *parser)
+{
+	fzl_node_t *ast = parse_program(parser);
+	if (!ast) return -1;
+
+	parser->root = ast;
+	return 0;
+}
+
+// --- Parse zone end ---
+
+char *fzl_parser_geterror()
+{
+	return last_err;
+}
+
+void fzl_freenode(fzl_node_t *node)
+{
+	if (!node) return;
+	if (node->token) fzl_freetoken(node->token);
+	free(node);
+}
+
+void fzl_freeparser(fzl_parser_t *parser)
+{
+	if (!parser) return;
+	free(parser);
+}
+

src/compiler/fzl_reader.c

@@ -0,0 +1,202 @@
+#include <compiler/fzl_reader.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// Last error from reader
+static char last_err[256] = {0};
+
+fzl_reader_t *fzl_mkreader(char *source)
+{
+	fzl_reader_ll_t *first_node = malloc(sizeof(fzl_reader_ll_t));
+	first_node->token = NULL;
+	first_node->next = NULL;
+
+	fzl_reader_t *reader = malloc(sizeof(fzl_reader_t));
+	reader->first = first_node;
+	reader->last = first_node;
+	reader->source = source;
+	reader->position = 0;
+	reader->at_end = true;
+
+	return reader;
+}
+
+// --- Read zone ---
+
+// Try parse name
+fzl_token_t *parse_name(fzl_reader_t *reader)
+{
+	size_t lpos = reader->position;
+	size_t size = 0;
+
+	do
+	{
+		lpos++;
+		size++;
+	}
+	while ((reader->source[lpos] >= 'A' && reader->source[lpos] <= 'Z')
+			|| (reader->source[lpos] >= 'a' && reader->source[lpos] <= 'z'));
+
+	char *value = malloc(size);
+	memcpy(value, reader->source + reader->position, size);
+
+	size_t start = reader->position;
+	reader->position += size;
+
+	fzl_token_t *tok = fzl_mktoken(
+		FZL_TOKTYPE_NAME,
+		start,
+		reader->position - 1,
+		value,
+		size
+	);
+	free(value);
+
+	return tok;
+}
+
+// Try parse string literal
+fzl_token_t *parse_string_literal(fzl_reader_t *reader)
+{
+	size_t lpos = reader->position;
+	size_t size = 0;
+
+	if (reader->source[lpos] != '\'' && reader->source[lpos] != '"')
+		return NULL;
+
+	char open_symbol = reader->source[lpos];
+
+	do
+	{
+		lpos++;
+		size++;
+	}
+	while (reader->source[lpos] != open_symbol);
+	size++;
+
+	if (size == 2)
+	{
+		fzl_token_t *tok = fzl_mktoken(
+			FZL_TOKTYPE_STR_LITERAL,
+			reader->position,
+			reader->position + 1,
+			"",
+			0
+		);
+		return tok;
+	}
+
+	char *value = malloc(size - 2);
+	memcpy(value, reader->source + reader->position + 1, size - 2);
+	fzl_token_t *tok = fzl_mktoken(
+		FZL_TOKTYPE_STR_LITERAL,
+		reader->position,
+		lpos,
+		value,
+		size - 2
+	);
+	free(value);
+
+	reader->position += size;
+
+	return tok;
+}
+
+int fzl_read(fzl_reader_t *reader)
+{
+	const size_t len = strlen(reader->source);
+	if (reader->position >= len) return 0;
+	
+	do {
+		size_t p = reader->position;
+		fzl_token_t *token;
+		char curr = reader->source[p];
+
+		if (curr == '\n' || curr == ' ' || curr == '\r')
+		{
+			reader->position++;
+			continue;
+		}
+
+		if (curr == '\'' || curr == '"')
+			token = parse_string_literal(reader);
+		else if (curr == '(')
+		{
+			token = fzl_mktoken(
+				FZL_TOKTYPE_LPAIR,
+				p, p, NULL, 0);
+			reader->position++;
+		}
+		else if (curr == ')')
+		{
+			token = fzl_mktoken(
+				FZL_TOKTYPE_RPAIR,
+				p, p, NULL, 0);
+			reader->position++;
+		}
+		else if (curr >= 'A' && curr <= 'Z' ||
+						 curr >= 'a' && curr <= 'z')
+			token = parse_name(reader);
+		else
+		{
+			sprintf(last_err, "undefined token '%c'", curr);
+			return -1;
+		}
+
+		if (!token)
+		{
+			size_t line = 1, linepos = 1;
+			for (size_t i = 0; i < p; i++)
+			{
+				char sym = reader->source[i];
+				if (sym == '\n')
+				{
+					linepos = 1;
+					line++;
+				} else linepos++;
+			}
+
+			sprintf(last_err, "error parsing token (position %ld of line %ld)", line, linepos);
+			return -1;
+		}
+
+		fzl_reader_ll_t *tok_node = malloc(sizeof(fzl_reader_ll_t));
+		tok_node->token = token;
+		
+		reader->last->next = tok_node;
+		reader->last = tok_node;
+	} while (reader->position < len);
+
+	reader->at_end = true;
+	return 0;
+}
+
+// --- Read zone end ---
+
+char *fzl_reader_geterror()
+{
+	return last_err;
+}
+
+void free_node_and_next(fzl_reader_ll_t *node)
+{
+	if (!node) return;
+	if (node->next)
+		free_node_and_next(node->next);
+	
+	if (node->token)
+		fzl_freetoken(node->token);
+
+	free(node);
+}
+
+void fzl_freereader(fzl_reader_t *reader)
+{
+	if (!reader) return;
+	if (reader->first)
+		free_node_and_next(reader->first);
+	
+	free(reader);
+}
+

src/compiler/fzl_token.c

@@ -0,0 +1,29 @@
+#include <compiler/fzl_token.h>
+#include <stdlib.h>
+#include <string.h>
+
+fzl_token_t *fzl_mktoken(
+	const fzl_toktype_t type, uint16_t from, uint16_t to,
+	void *value, size_t value_len)
+{
+	void *valholder = malloc(value_len);
+	memcpy(valholder, value, value_len);
+
+	fzl_token_t *token = malloc(sizeof(fzl_token_t));
+	token->type = type;
+	token->i0 = from;
+	token->i1 = to;
+	token->store_value = value_len > 0;
+	token->value = valholder;
+	token->value_sz = value_len;
+
+	return token;
+}
+
+// Free allocated token
+void fzl_freetoken(fzl_token_t *tok)
+{
+	if (tok->store_value) free(tok->value);
+	free(tok);
+}
+

src/compiler/main.c

@@ -0,0 +1,80 @@
+#include <stdio.h>
+#include <string.h>
+#include <compiler/fzl_reader.h>
+#include <compiler/fzl_parser.h>
+#include <compiler/fzl_debug.h>
+
+int main(int argc, char** argv) {
+	if (argc < 2)
+	{
+		printf("No files provided >:(\n");
+		return -1;
+	}
+	else if (argc > 2)
+	{
+		printf("warn: flags are not supported yet\n");
+	}
+
+	printf("> analyze\n-> reading file...");
+
+	char source[8192];
+	FILE *fd = fopen(argv[argc - 1], "r");
+	if (!fd)
+	{
+		printf("failed to open file '%s'\n", argv[argc-1]);
+		return -1;
+	}
+	
+	fseek(fd, 0L, SEEK_END);
+	size_t fsz = ftell(fd);
+	rewind(fd);
+
+	fread(source, fsz, 1, fd);
+	fclose(fd);
+	printf("ok!\n");
+
+	printf("-> tokenizing...");
+	fzl_reader_t *rd = fzl_mkreader(source);
+	int result = fzl_read(rd);
+	if (result != 0)
+	{
+		printf("error: %s\n", fzl_reader_geterror());
+		return -1;
+	}
+	
+	printf("ok!\n");
+
+	fzl_reader_ll_t *entry = rd->first;
+	printf("--> structure: ");
+	while (entry)
+	{
+		if (entry->token)
+		{
+			if (entry->token->store_value)
+				printf("<TOK:%d:%s>", entry->token->type, (char*)entry->token->value);
+			else
+				printf("<TOK:%d>", entry->token->type);
+		}
+
+		entry = entry->next;
+	}
+
+	printf("\n");
+
+	printf("-> parsing...");
+	fzl_parser_t *parser = fzl_mkparser(rd->first);
+	result = fzl_parse(parser);
+	if (result != 0)
+	{
+		printf("error: %s\n", fzl_parser_geterror());
+		return -1;
+	}
+
+	printf("ok!\n");
+	fzl_debug_print_ast(parser->root);
+
+	fzl_freereader(rd);
+	fzl_freeparser(parser);
+	return 0;
+}
+