diff --git a/grammars/verilogPreprocLexer.g4 b/grammars/verilogPreprocLexer.g4 index 8fb4adec..59cc0ee2 100644 --- a/grammars/verilogPreprocLexer.g4 +++ b/grammars/verilogPreprocLexer.g4 @@ -359,11 +359,15 @@ mode KEYWOORDS_MODE; // include argument processing mode INCLUDE_MODE; INCLUDE_MODE_STR - : STR -> type(STR),popMode + : STR -> type(STR) ; + INCLUDE_MODE_STR_CHEVRONS - : '<' ( ~('\\'|'>') )* '>' -> type(STR),popMode + : '<' ( ~('\\'|'>') )* '>' -> type(STR) ; + INCLUDE_LINE_COMMENT : LINE_COMMENT -> popMode,type(LINE_COMMENT),channel(CH_LINE_COMMENT); + INCLUDE_MODE_NEW_LINE: CRLF -> type(NEW_LINE),popMode,skip; + INCLUDE_MODE_MACRO_ENTER: '`' -> popMode,pushMode(DIRECTIVE_MODE),skip; INCLUDE_MODE_WS : WS ->skip; diff --git a/hdlConvertor/toPy.cpp b/hdlConvertor/toPy.cpp index d250a717..778f5edc 100644 --- a/hdlConvertor/toPy.cpp +++ b/hdlConvertor/toPy.cpp @@ -190,6 +190,8 @@ PyObject* ToPy::toPy(const CodePosition o) { return nullptr; if (toPy_property(py_inst, "stop_column", o.stop_column)) return nullptr; + if (toPy_property(py_inst, "file_name", o.file_name)) + return nullptr; return py_inst; } diff --git a/include/hdlConvertor/hdlAst/codePosition.h b/include/hdlConvertor/hdlAst/codePosition.h index 07e0e058..eb0f66d6 100644 --- a/include/hdlConvertor/hdlAst/codePosition.h +++ b/include/hdlConvertor/hdlAst/codePosition.h @@ -2,6 +2,8 @@ #include #include +#include + namespace hdlConvertor { namespace hdlAst { @@ -13,22 +15,34 @@ namespace hdlAst { class CodePosition { public: static constexpr size_t INVALID = std::numeric_limits::max(); + static constexpr char INVALID_STR[] = ""; size_t start_line; size_t stop_line; size_t start_column; size_t stop_column; + std::string file_name; CodePosition(); CodePosition(size_t startLine, size_t stopLine, size_t startColumn, - size_t stopColumn); + size_t stopColumn, std::string fileName); template void update_from_elem(ELEM_T *elem) { - start_line = elem->getStart()->getLine(); - stop_line = elem->getStop()->getLine(); - start_column = elem->getStart()->getCharPositionInLine() + 1; - stop_column = elem->getStop()->getCharPositionInLine() + - (elem->getStop()->getStopIndex() - elem->getStop()->getStartIndex()) + 1; + /* + * Cast antlr4::CommonToken to our custum hdlConvertorToken + * So we can retrive the file name information of origin of + * current token. + */ + hdlConvertor::hdlConvertorToken * tokenStart_ptr = dynamic_cast(elem->getStart()); + hdlConvertor::hdlConvertorToken * tokenStop_ptr = dynamic_cast(elem->getStop()); + + /* pick up data from token */ + start_line = tokenStart_ptr->getLine(); + stop_line = tokenStop_ptr->getLine(); + start_column = tokenStart_ptr->getCharPositionInLine() + 1; + stop_column = tokenStop_ptr->getCharPositionInLine() + + (tokenStop_ptr->getStopIndex() - tokenStop_ptr->getStartIndex()) + 1; + file_name = tokenStart_ptr->getSrcName(); } bool isKnown() const; }; diff --git a/include/hdlConvertor/hdlConvertor.h b/include/hdlConvertor/hdlConvertor.h index 66c0c180..faf031ab 100644 --- a/include/hdlConvertor/hdlConvertor.h +++ b/include/hdlConvertor/hdlConvertor.h @@ -12,6 +12,7 @@ #include #include #include +#include namespace hdlConvertor { diff --git a/include/hdlConvertor/hdlConvertorToken.h b/include/hdlConvertor/hdlConvertorToken.h new file mode 100644 index 00000000..64e7720e --- /dev/null +++ b/include/hdlConvertor/hdlConvertorToken.h @@ -0,0 +1,23 @@ +#pragma once + +#include + +namespace hdlConvertor { + +class hdlConvertorToken : public antlr4::CommonToken { + public: + hdlConvertorToken(size_t type); + hdlConvertorToken(std::pair source, size_t type, size_t channel, size_t start, size_t stop); + hdlConvertorToken(size_t type, const std::string &text); + hdlConvertorToken(antlr4::Token *oldToken); + + void setSrcName(const std::string fileName); + std::string getSrcName(); + std::string toString() const override; + + private: + std::string srcName; + +}; + +} diff --git a/include/hdlConvertor/hdlConvertorTokenFactory.h b/include/hdlConvertor/hdlConvertorTokenFactory.h new file mode 100644 index 00000000..886ad04b --- /dev/null +++ b/include/hdlConvertor/hdlConvertorTokenFactory.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include +#include +namespace hdlConvertor { + +class hdlConvertorTokenFactory : public antlr4::CommonTokenFactory { + public: + static const std::unique_ptr> DEFAULT; + + hdlConvertorTokenFactory(bool copy_text); + hdlConvertorTokenFactory(); + void setFileLineMap( verilog_pp::FileLineMap * file_line_map); + + std::unique_ptr create( + std::pair source, + size_t type, + const std::string &text, + size_t channel, + size_t start, + size_t stop, + size_t line, + size_t charPositionInLine) override; + + std::unique_ptr create( + size_t type, + const std::string &text) override; + + + private: + static verilog_pp::FileLineMap *_file_line_map; + +}; + +} diff --git a/src/hdlAst/codePosition.cpp b/src/hdlAst/codePosition.cpp index 7c9f1fbf..105dfde6 100644 --- a/src/hdlAst/codePosition.cpp +++ b/src/hdlAst/codePosition.cpp @@ -4,20 +4,21 @@ namespace hdlConvertor { namespace hdlAst { CodePosition::CodePosition() : - CodePosition(INVALID, INVALID, INVALID, INVALID) { + CodePosition(INVALID, INVALID, INVALID, INVALID,INVALID_STR) { } CodePosition::CodePosition(size_t startLine, size_t stopLine, size_t startColumn, - size_t stopColumn) { + size_t stopColumn,std::string fileName) { this->start_line = startLine; this->stop_line = stopLine; this->start_column = startColumn; this->stop_column = stopColumn; + this->file_name = fileName; } bool CodePosition::isKnown() const { return start_line != INVALID || stop_line != INVALID || start_column != INVALID - || stop_column != INVALID; + || stop_column != INVALID || file_name != INVALID_STR; } } diff --git a/src/hdlConvertor.cpp b/src/hdlConvertor.cpp index 9af35b77..b8e7c843 100644 --- a/src/hdlConvertor.cpp +++ b/src/hdlConvertor.cpp @@ -25,6 +25,20 @@ class VHDLParserContainer: public iParserContainer { using iParserContainer::iParserContainer; virtual void parseFn() override { + + /* + * Customize Lexer with our hdlConvertorToken factory + * This allow us to enrich antlr4::CommonToken with filename on + * each token, then spread this kind of detail everywhere token are + * use. + * Dummy structure for vhdl, since no processing step exist. + * */ + verilog_pp::FileLineMap dummy_file_line_map; + hdlConvertorTokenFactory factory; + factory.setFileLineMap(&dummy_file_line_map); + lexer.get()->setTokenFactory(factory.DEFAULT.get()); + + vhdl_antlr::vhdlParser::Design_fileContext *tree = antlrParser->design_file(); syntaxErrLogger.check_errors(); // Throw exception if errors @@ -56,6 +70,14 @@ class SVParserContainer: public iParserContainer_parse(input_for_parser, hierarchyOnly); @@ -77,8 +99,28 @@ class SVParserContainer: public iParserContainerlanguage_version = lang; + + /* + * Customize Lexer with our hdlConvertorToken factory + * This allow us to enrich antlr4::CommonToken with filename on + * each token, then spread this kind of detail everywhere token are + * use + * */ + hdlConvertorTokenFactory factory; + factory.setFileLineMap(&file_line_map); + lexer.get()->setTokenFactory(factory.DEFAULT.get()); + + /* Debug print hdlConvertorToken list + tokens.get()->fill(); + for (auto token : tokens.get()->getTokens()) { + std::cout << token->toString() << std::endl; + } + */ + sv2017_antlr::sv2017Parser::Source_textContext *tree = antlrParser->source_text(); + + syntaxErrLogger.check_errors(); // Throw exception if errors hdlParser->visitSource_text(tree); } diff --git a/src/hdlConvertorToken.cpp b/src/hdlConvertorToken.cpp new file mode 100644 index 00000000..9d6fba90 --- /dev/null +++ b/src/hdlConvertorToken.cpp @@ -0,0 +1,38 @@ +#include + + +namespace hdlConvertor { + +using namespace std; + +hdlConvertorToken::hdlConvertorToken(size_t type) : CommonToken(type) { +} + +hdlConvertorToken::hdlConvertorToken(pair source, size_t type, size_t channel, size_t start, size_t stop) : + CommonToken(source,type,channel,start,stop),srcName("cornichon") { + +} + +hdlConvertorToken::hdlConvertorToken(size_t type, const std::string &text) : + CommonToken(type,text),srcName("corniChon") { +} + +hdlConvertorToken::hdlConvertorToken(antlr4::Token *oldToken) : + CommonToken(oldToken) { +} + + +void hdlConvertorToken::setSrcName(const string fileName) { + srcName = fileName; +} + +std::string hdlConvertorToken::getSrcName() { + return srcName; +} + +string hdlConvertorToken::toString() const { + string rt_value = srcName + CommonToken::toString(); + return rt_value; +} + +} diff --git a/src/hdlConvertorTokenFactory.cpp b/src/hdlConvertorTokenFactory.cpp new file mode 100644 index 00000000..dd80c03a --- /dev/null +++ b/src/hdlConvertorTokenFactory.cpp @@ -0,0 +1,84 @@ +#include + +namespace hdlConvertor { + +using namespace std; + +const std::unique_ptr> hdlConvertorTokenFactory::DEFAULT(new hdlConvertorTokenFactory); + +verilog_pp::FileLineMap* hdlConvertorTokenFactory::_file_line_map = nullptr; + + +hdlConvertorTokenFactory::hdlConvertorTokenFactory( + bool copyText) : + CommonTokenFactory(copyText) { +} + +hdlConvertorTokenFactory::hdlConvertorTokenFactory() { +} + +void hdlConvertorTokenFactory::setFileLineMap( verilog_pp::FileLineMap * file_line_map){ + hdlConvertorTokenFactory::_file_line_map = file_line_map; +} + +std::unique_ptr +hdlConvertorTokenFactory::create( + std::pair source, + size_t type, + const std::string &text, + size_t channel, + size_t start, + size_t stop, + size_t line, + size_t charPositionInLine) { + + bool found = false; + + std::unique_ptr t(new hdlConvertorToken( + source, type, channel, start, stop + ) + ); + + hdlConvertorToken * ptr = dynamic_cast(t.get()); + + std::vector::iterator fileLineMap_iter; + fileLineMap_iter = (*hdlConvertorTokenFactory::_file_line_map).begin(); + while (fileLineMap_iter != (*hdlConvertorTokenFactory::_file_line_map).end() && found==false) { + + if (line == (fileLineMap_iter->line+1)) { + found = true; + ptr->setLine(fileLineMap_iter->line_override+1); + ptr->setSrcName(fileLineMap_iter->file_override); + } + + fileLineMap_iter++; + } + if (found == false) { + ptr->setLine(line); + /* line not found ! For time beeing keep original data + std::cout << "Drama! " + << text + << "line: "<< line << " not found" << std::endl; + */ + } + ptr->setCharPositionInLine(charPositionInLine); + if (text != "") { + ptr->setText(text); + } else if (copyText && source.second != nullptr) { + ptr->setText(source.second->getText(antlr4::misc::Interval(start, stop))); + } + + return t; +} + + + +std::unique_ptr +hdlConvertorTokenFactory::create( + size_t type, + const std::string &text) { + return std::unique_ptr(new hdlConvertorToken(type, text)); + +} + +} diff --git a/src/meson.build b/src/meson.build index a139be7b..59fb4a3a 100644 --- a/src/meson.build +++ b/src/meson.build @@ -8,6 +8,8 @@ hdlConvertor_core_src = [ 'conversion_exception.cpp', 'universal_fs.cpp', 'encodingConversions.cpp', + 'hdlConvertorToken.cpp', + 'hdlConvertorTokenFactory.cpp' ] hdlConvertor_core_src += run_command(py3, '../utils/rec_glob.py', './hdlAst', '*.cpp', check:true)\ .stdout().strip().split('\n') diff --git a/src/toString.cpp b/src/toString.cpp index e6743bfb..2eb5354b 100644 --- a/src/toString.cpp +++ b/src/toString.cpp @@ -172,6 +172,7 @@ void ToString::dump(const CodePosition * o, int indent) { dump_size_t("stopLine", o->stop_line); dump_size_t("startColumn", o->start_column); dump_size_t("stopColumn", o->stop_column); + dump_size_t("fileName",o->file_name); indent -= INDENT_INCR; mkIndent(indent) << "}"; } diff --git a/src/verilogPreproc/out_buffer.cpp b/src/verilogPreproc/out_buffer.cpp index 68d863e5..522f2ba1 100644 --- a/src/verilogPreproc/out_buffer.cpp +++ b/src/verilogPreproc/out_buffer.cpp @@ -60,7 +60,7 @@ VerilogPreprocOutBuffer& VerilogPreprocOutBuffer::operator<<( VerilogPreprocOutBuffer& VerilogPreprocOutBuffer::operator<<( const VerilogPreprocOutBuffer &other) { - (*this) << endl; + //(*this) << endl; (*static_cast(this)) << other.str(); for (auto &i : other.file_line_map) { FileLineMapItem i_new(i.line + output_line, i.file_override, diff --git a/tests/sv_pp/expected/debug_macro.txt b/tests/sv_pp/expected/debug_macro.txt index a1a5103d..8a8ee519 100644 --- a/tests/sv_pp/expected/debug_macro.txt +++ b/tests/sv_pp/expected/debug_macro.txt @@ -5,6 +5,4 @@ - - $display(">> %s, %d: %s", "sv_pp/src/debug_macro.txt", 3, "msg") diff --git a/tests/sv_pp/expected/include_many_dir/dir0-a/dir1-a/spec_incdir.txt b/tests/sv_pp/expected/include_many_dir/dir0-a/dir1-a/spec_incdir.txt index 807a1ca1..1124e96b 100644 --- a/tests/sv_pp/expected/include_many_dir/dir0-a/dir1-a/spec_incdir.txt +++ b/tests/sv_pp/expected/include_many_dir/dir0-a/dir1-a/spec_incdir.txt @@ -1,2 +1 @@ - "good content" \ No newline at end of file diff --git a/tests/sv_pp/expected/include_many_dir/dir0-a/local_include_higher_priority.txt b/tests/sv_pp/expected/include_many_dir/dir0-a/local_include_higher_priority.txt index 807a1ca1..1124e96b 100644 --- a/tests/sv_pp/expected/include_many_dir/dir0-a/local_include_higher_priority.txt +++ b/tests/sv_pp/expected/include_many_dir/dir0-a/local_include_higher_priority.txt @@ -1,2 +1 @@ - "good content" \ No newline at end of file diff --git a/tests/sv_pp/expected/include_many_dir/from_subdirectory.txt b/tests/sv_pp/expected/include_many_dir/from_subdirectory.txt index 8528768f..d1f22783 100644 --- a/tests/sv_pp/expected/include_many_dir/from_subdirectory.txt +++ b/tests/sv_pp/expected/include_many_dir/from_subdirectory.txt @@ -1,6 +1 @@ - -"file2-a.txt" - -"file2-a.txt" - -"file2-a.txt" +"file2-a.txt""file2-a.txt""file2-a.txt" \ No newline at end of file diff --git a/tests/sv_pp/expected/include_many_dir/from_subdirectory2.txt b/tests/sv_pp/expected/include_many_dir/from_subdirectory2.txt new file mode 100644 index 00000000..53e98acd --- /dev/null +++ b/tests/sv_pp/expected/include_many_dir/from_subdirectory2.txt @@ -0,0 +1,3 @@ +"file2-a.txt" +"file2-a.txt" +"file2-a.txt" diff --git a/tests/sv_pp/expected/include_many_dir/transitive.txt b/tests/sv_pp/expected/include_many_dir/transitive.txt index a7c0eed4..92cde8dc 100644 --- a/tests/sv_pp/expected/include_many_dir/transitive.txt +++ b/tests/sv_pp/expected/include_many_dir/transitive.txt @@ -1,8 +1,4 @@ - "transitive-0" - "transitive-1" - "transitive-2" - "transitive-3" \ No newline at end of file diff --git a/tests/sv_pp/expected/include_same_dir/basic_include.txt b/tests/sv_pp/expected/include_same_dir/basic_include.txt index 12fb4c89..5e2ef389 100644 --- a/tests/sv_pp/expected/include_same_dir/basic_include.txt +++ b/tests/sv_pp/expected/include_same_dir/basic_include.txt @@ -1,4 +1,3 @@ - "a.vh" diff --git a/tests/sv_pp/expected/include_same_dir/basic_include2times.txt b/tests/sv_pp/expected/include_same_dir/basic_include2times.txt index dd0b6a63..5e2ef389 100644 --- a/tests/sv_pp/expected/include_same_dir/basic_include2times.txt +++ b/tests/sv_pp/expected/include_same_dir/basic_include2times.txt @@ -1,6 +1,3 @@ - "a.vh" - - diff --git a/tests/sv_pp/src/include_many_dir/dir0-a/dir1-a/file2-b.txt b/tests/sv_pp/src/include_many_dir/dir0-a/dir1-a/file2-b.txt new file mode 100644 index 00000000..ba5cbc4f --- /dev/null +++ b/tests/sv_pp/src/include_many_dir/dir0-a/dir1-a/file2-b.txt @@ -0,0 +1 @@ +"file2-a.txt" diff --git a/tests/sv_pp/src/include_many_dir/from_subdirectory2.txt b/tests/sv_pp/src/include_many_dir/from_subdirectory2.txt new file mode 100644 index 00000000..dbf9d3d1 --- /dev/null +++ b/tests/sv_pp/src/include_many_dir/from_subdirectory2.txt @@ -0,0 +1,3 @@ +`include "dir0-a/dir1-a/file2-b.txt" +`include "./dir0-a/dir1-a/file2-b.txt" +`include "./dir0-a/../dir0-a/dir1-a/file2-b.txt" diff --git a/tests/test_verilog_preproc_include.py b/tests/test_verilog_preproc_include.py index 92187d68..a906c78e 100644 --- a/tests/test_verilog_preproc_include.py +++ b/tests/test_verilog_preproc_include.py @@ -43,6 +43,9 @@ def test_basic_include2times(self): def test_from_subdirectory(self): self.run_test(["include_many_dir", "from_subdirectory.txt"], []) + + def test_from_subdirectory2(self): + self.run_test(["include_many_dir", "from_subdirectory2.txt"], []) def test_local_include_higher_priority(self): self.run_test(