[WIP] Fix issue 166

grammars/verilogPreprocLexer.g4

@@ -359,11 +359,15 @@ mode KEYWOORDS_MODE;
 // include argument processing
 mode INCLUDE_MODE;
     INCLUDE_MODE_STR
-        :  STR -> type(STR),popMode
+        :  STR -> type(STR)
         ;
+
     INCLUDE_MODE_STR_CHEVRONS
-        :  '<' ( ~('\\'|'>') )* '>' -> type(STR),popMode
+        :  '<' ( ~('\\'|'>') )* '>' -> type(STR)
         ;
+    INCLUDE_LINE_COMMENT : LINE_COMMENT -> popMode,type(LINE_COMMENT),channel(CH_LINE_COMMENT);
+    INCLUDE_MODE_NEW_LINE: CRLF -> type(NEW_LINE),popMode,skip;
+
     INCLUDE_MODE_MACRO_ENTER: '`' -> popMode,pushMode(DIRECTIVE_MODE),skip;
     INCLUDE_MODE_WS : WS ->skip;
 

hdlConvertor/toPy.cpp

@@ -190,6 +190,8 @@ PyObject* ToPy::toPy(const CodePosition o) {
 		return nullptr;
 	if (toPy_property(py_inst, "stop_column", o.stop_column))
 		return nullptr;
+	if (toPy_property(py_inst, "file_name", o.file_name))
+		return nullptr;
 	return py_inst;
 }
 

include/hdlConvertor/hdlAst/codePosition.h

@@ -2,6 +2,8 @@
 #include <stddef.h>
 #include <limits>
 
+#include <hdlConvertor/hdlConvertorToken.h>
+
 namespace hdlConvertor {
 namespace hdlAst {
 
@@ -13,22 +15,34 @@ namespace hdlAst {
 class CodePosition {
 public:
 	static constexpr size_t INVALID = std::numeric_limits<size_t>::max();
+	static constexpr char INVALID_STR[] = "<INVSTR>";
 
 	size_t start_line;
 	size_t stop_line;
 	size_t start_column;
 	size_t stop_column;
+	std::string file_name;
 
 	CodePosition();
 	CodePosition(size_t startLine, size_t stopLine, size_t startColumn,
-			size_t stopColumn);
+			size_t stopColumn, std::string fileName);
 	template<class ELEM_T>
 	void update_from_elem(ELEM_T *elem) {
-		start_line = elem->getStart()->getLine();
-		stop_line = elem->getStop()->getLine();
-		start_column = elem->getStart()->getCharPositionInLine() + 1;
-		stop_column = elem->getStop()->getCharPositionInLine() +
-				(elem->getStop()->getStopIndex() - elem->getStop()->getStartIndex()) + 1;
+		/*
+		 * Cast antlr4::CommonToken to our custum hdlConvertorToken
+		 * So we can retrive the file name information of origin of
+		 * current token.
+		 */
+		hdlConvertor::hdlConvertorToken * tokenStart_ptr = dynamic_cast<hdlConvertor::hdlConvertorToken *>(elem->getStart());
+		hdlConvertor::hdlConvertorToken * tokenStop_ptr = dynamic_cast<hdlConvertor::hdlConvertorToken *>(elem->getStop());
+
+		/* pick up data from token */
+		start_line = tokenStart_ptr->getLine();
+		stop_line = tokenStop_ptr->getLine();
+		start_column = tokenStart_ptr->getCharPositionInLine() + 1;
+		stop_column = tokenStop_ptr->getCharPositionInLine() +
+				(tokenStop_ptr->getStopIndex() - tokenStop_ptr->getStartIndex()) + 1;
+		file_name = tokenStart_ptr->getSrcName();
 	}
 	bool isKnown() const;
 };

include/hdlConvertor/hdlConvertor.h

@@ -12,6 +12,7 @@
 #include <hdlConvertor/parserContainer.h>
 #include <hdlConvertor/hdlAst/hdlContext.h>
 #include <hdlConvertor/verilogPreproc/verilogPreproc.h>
+#include <hdlConvertor/hdlConvertorTokenFactory.h>
 
 namespace hdlConvertor {
 

include/hdlConvertor/hdlConvertorToken.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <antlr4-runtime.h>
+
+namespace hdlConvertor {
+
+class hdlConvertorToken : public antlr4::CommonToken {
+	public:
+	hdlConvertorToken(size_t type);
+    	hdlConvertorToken(std::pair<antlr4::TokenSource*, antlr4::CharStream*> source, size_t type, size_t channel, size_t start, size_t stop);
+	hdlConvertorToken(size_t type, const std::string &text);
+	hdlConvertorToken(antlr4::Token *oldToken);
+
+	void setSrcName(const std::string fileName);
+	std::string getSrcName();
+	std::string toString() const override;
+
+	private:
+		std::string srcName;
+
+};
+
+}

include/hdlConvertor/hdlConvertorTokenFactory.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include <hdlConvertor/verilogPreproc/file_line_map.h>
+#include <hdlConvertor/hdlConvertorToken.h>
+#include <antlr4-runtime.h>
+namespace hdlConvertor {
+
+class hdlConvertorTokenFactory : public antlr4::CommonTokenFactory {
+	public:
+        static const std::unique_ptr<antlr4::TokenFactory<antlr4::CommonToken>> DEFAULT;
+
+	hdlConvertorTokenFactory(bool copy_text);
+	hdlConvertorTokenFactory();
+	void setFileLineMap( verilog_pp::FileLineMap * file_line_map);
+
+	std::unique_ptr<antlr4::CommonToken> create(
+			std::pair<antlr4::TokenSource*, antlr4::CharStream*> source,
+			size_t type,
+      			const std::string &text,
+			size_t channel,
+			size_t start,
+			size_t stop,
+			size_t line,
+			size_t charPositionInLine) override;
+
+	std::unique_ptr<antlr4::CommonToken> create(
+			size_t type,
+			const std::string &text) override;
+
+
+	private:
+	static verilog_pp::FileLineMap *_file_line_map;
+
+};
+
+}

src/hdlAst/codePosition.cpp

@@ -4,20 +4,21 @@ namespace hdlConvertor {
 namespace hdlAst {
 
 CodePosition::CodePosition() :
-		CodePosition(INVALID, INVALID, INVALID, INVALID) {
+		CodePosition(INVALID, INVALID, INVALID, INVALID,INVALID_STR) {
 }
 
 CodePosition::CodePosition(size_t startLine, size_t stopLine, size_t startColumn,
-		size_t stopColumn) {
+		size_t stopColumn,std::string fileName) {
 	this->start_line = startLine;
 	this->stop_line = stopLine;
 	this->start_column = startColumn;
 	this->stop_column = stopColumn;
+	this->file_name = fileName;
 }
 
 bool CodePosition::isKnown() const {
 	return start_line != INVALID || stop_line != INVALID || start_column != INVALID
-			|| stop_column != INVALID;
+			|| stop_column != INVALID || file_name != INVALID_STR;
 }
 
 }

src/hdlConvertor.cpp

@@ -25,6 +25,20 @@ class VHDLParserContainer: public iParserContainer<vhdl_antlr::vhdlLexer,
 		vhdl_antlr::vhdlParser, vhdl::VhdlDesignFileParser> {
 	using iParserContainer::iParserContainer;
 	virtual void parseFn() override {
+
+		/*
+		 * Customize Lexer with our hdlConvertorToken factory
+		 * This allow us to enrich antlr4::CommonToken with filename on
+		 * each token, then spread this kind of detail everywhere token are
+		 * use.
+		 * Dummy structure for vhdl, since no processing step exist.
+		 * */
+		verilog_pp::FileLineMap dummy_file_line_map;
+		hdlConvertorTokenFactory factory;
+		factory.setFileLineMap(&dummy_file_line_map);
+		lexer.get()->setTokenFactory(factory.DEFAULT.get());
+
+
 		vhdl_antlr::vhdlParser::Design_fileContext *tree =
 				antlrParser->design_file();
 		syntaxErrLogger.check_errors(); // Throw exception if errors
@@ -56,6 +70,14 @@ class SVParserContainer: public iParserContainer<sv2017_antlr::sv2017Lexer,
 		string preprocessed_code = preprocess_res.str();
 		file_line_map = preprocess_res.file_line_map;
 
+		/* Debug print FileLineMap data
+		std::cout << "File_line_map" << std::endl;
+		for (hdlConvertor::verilog_pp::FileLineMapItem item : file_line_map){
+		  std::cout << item.line << ' ' << item.file_override.c_str() << ' ' << item.line_override << std::endl;
+		}
+		*/
+
+
 		antlr4::ANTLRInputStream input_for_parser(preprocessed_code);
 		input_for_parser.name = file_name.u8string();
 		this->_parse(input_for_parser, hierarchyOnly);
@@ -77,8 +99,28 @@ class SVParserContainer: public iParserContainer<sv2017_antlr::sv2017Lexer,
 
 	virtual void parseFn() override {
 		lexer->language_version = lang;
+
+		/*
+		 * Customize Lexer with our hdlConvertorToken factory
+		 * This allow us to enrich antlr4::CommonToken with filename on
+		 * each token, then spread this kind of detail everywhere token are
+		 * use
+		 * */
+		hdlConvertorTokenFactory factory;
+		factory.setFileLineMap(&file_line_map);
+		lexer.get()->setTokenFactory(factory.DEFAULT.get());
+
+		/* Debug print hdlConvertorToken list
+		tokens.get()->fill();
+		for (auto token : tokens.get()->getTokens()) {
+    			std::cout << token->toString() << std::endl;
+  		}
+		*/
+
 		sv2017_antlr::sv2017Parser::Source_textContext *tree =
 				antlrParser->source_text();
+
+
 		syntaxErrLogger.check_errors(); // Throw exception if errors
 		hdlParser->visitSource_text(tree);
 	}

src/hdlConvertorToken.cpp

@@ -0,0 +1,38 @@
+#include <hdlConvertor/hdlConvertorToken.h>
+
+
+namespace hdlConvertor {
+
+using namespace std;
+
+hdlConvertorToken::hdlConvertorToken(size_t type) : CommonToken(type) {
+}
+
+hdlConvertorToken::hdlConvertorToken(pair<antlr4::TokenSource*, antlr4::CharStream*> source, size_t type, size_t channel, size_t start, size_t stop) :
+	CommonToken(source,type,channel,start,stop),srcName("cornichon") {
+
+}
+
+hdlConvertorToken::hdlConvertorToken(size_t type, const std::string &text) :
+	CommonToken(type,text),srcName("corniChon") {
+}
+
+hdlConvertorToken::hdlConvertorToken(antlr4::Token *oldToken) :
+	CommonToken(oldToken) {
+}
+
+
+void hdlConvertorToken::setSrcName(const string fileName) {
+	srcName = fileName;
+}
+
+std::string hdlConvertorToken::getSrcName() {
+	return srcName;
+}
+
+string hdlConvertorToken::toString() const {
+	string rt_value = srcName + CommonToken::toString();
+	return rt_value;
+}
+
+}

src/hdlConvertorTokenFactory.cpp

@@ -0,0 +1,84 @@
+#include <hdlConvertor/hdlConvertorTokenFactory.h>
+
+namespace hdlConvertor {
+
+using namespace std;
+
+const std::unique_ptr<antlr4::TokenFactory<antlr4::CommonToken>> hdlConvertorTokenFactory::DEFAULT(new hdlConvertorTokenFactory);
+
+verilog_pp::FileLineMap* hdlConvertorTokenFactory::_file_line_map = nullptr;
+
+
+hdlConvertorTokenFactory::hdlConvertorTokenFactory(
+		bool copyText) :
+	CommonTokenFactory(copyText) {
+}
+
+hdlConvertorTokenFactory::hdlConvertorTokenFactory() {
+}
+
+void hdlConvertorTokenFactory::setFileLineMap( verilog_pp::FileLineMap * file_line_map){
+	hdlConvertorTokenFactory::_file_line_map = file_line_map;
+}
+
+std::unique_ptr<antlr4::CommonToken>
+hdlConvertorTokenFactory::create(
+	std::pair<antlr4::TokenSource*, antlr4::CharStream*> source,
+	size_t type,
+      	const std::string &text,
+	size_t channel,
+	size_t start,
+	size_t stop,
+	size_t line,
+	size_t charPositionInLine) {
+
+	bool found = false;
+
+	std::unique_ptr<antlr4::CommonToken> t(new hdlConvertorToken(
+				source, type, channel, start, stop
+				)
+			);
+
+	hdlConvertorToken * ptr = dynamic_cast<hdlConvertorToken *>(t.get());
+
+	std::vector<verilog_pp::FileLineMapItem>::iterator fileLineMap_iter;
+	fileLineMap_iter = (*hdlConvertorTokenFactory::_file_line_map).begin();
+	while (fileLineMap_iter != (*hdlConvertorTokenFactory::_file_line_map).end() && found==false) {
+
+		if (line == (fileLineMap_iter->line+1)) {
+			found = true;
+			ptr->setLine(fileLineMap_iter->line_override+1);
+			ptr->setSrcName(fileLineMap_iter->file_override);
+		}
+
+		fileLineMap_iter++;
+	}
+	if (found ==  false) {
+		ptr->setLine(line);
+		/* line not found ! For time beeing keep original data
+		std::cout << "Drama! "
+			<< text
+			<< "line: "<< line << " not found" << std::endl;
+		*/
+	}
+	ptr->setCharPositionInLine(charPositionInLine);
+	if (text != "") {
+		ptr->setText(text);
+	} else if (copyText && source.second != nullptr) {
+		ptr->setText(source.second->getText(antlr4::misc::Interval(start, stop)));
+	}
+
+	return t;
+}
+
+
+
+std::unique_ptr<antlr4::CommonToken>
+hdlConvertorTokenFactory::create(
+	size_t type,
+	const std::string &text) {
+	return std::unique_ptr<antlr4::CommonToken>(new hdlConvertorToken(type, text));
+
+}
+
+}

src/meson.build

@@ -8,6 +8,8 @@ hdlConvertor_core_src = [
 	'conversion_exception.cpp',
 	'universal_fs.cpp',
 	'encodingConversions.cpp',
+        'hdlConvertorToken.cpp',
+        'hdlConvertorTokenFactory.cpp'
 ]
 hdlConvertor_core_src += run_command(py3, '../utils/rec_glob.py', './hdlAst', '*.cpp', check:true)\
 	.stdout().strip().split('\n')

src/toString.cpp

@@ -172,6 +172,7 @@ void ToString::dump(const CodePosition * o, int indent) {
 	dump_size_t("stopLine", o->stop_line);
 	dump_size_t("startColumn", o->start_column);
 	dump_size_t("stopColumn", o->stop_column);
+	dump_size_t("fileName",o->file_name);
 	indent -= INDENT_INCR;
 	mkIndent(indent) << "}";
 }

src/verilogPreproc/out_buffer.cpp

@@ -60,7 +60,7 @@ VerilogPreprocOutBuffer& VerilogPreprocOutBuffer::operator<<(
 
 VerilogPreprocOutBuffer& VerilogPreprocOutBuffer::operator<<(
 		const VerilogPreprocOutBuffer &other) {
-	(*this) << endl;
+	//(*this) << endl;
 	(*static_cast<stringstream*>(this)) << other.str();
 	for (auto &i : other.file_line_map) {
 		FileLineMapItem i_new(i.line + output_line, i.file_override,

tests/sv_pp/expected/debug_macro.txt

@@ -5,6 +5,4 @@
 
 
 
-
-
     $display(">> %s, %d: %s", "sv_pp/src/debug_macro.txt", 3, "msg")

tests/sv_pp/expected/include_many_dir/dir0-a/dir1-a/spec_incdir.txt

@@ -1,2 +1 @@
-
 "good content"