sugar/include/molasses/lexer.h

#pragma once
#include <vector>
#include <map>
#include <string>

namespace molasses {
	// We will always want symbols to be convertible to int for dictionary lookups
	struct symbol {
        int id;
        std::string file_name;
        int line;
        int column;
        [[no_unique_address]] bool is_string;

        operator int() const {
            return id;
        }
    };
	
	struct lexed_output {
		std::map<int, std::string> dictionary;
		std::vector<symbol> symbols;
	};
	
	lexed_output lex(const std::string & file_name,const std::string & source);
	lexed_output concatenate(const lexed_output& lhs, const lexed_output& rhs);
}