From 96b33b1936d37ef44b445ed0abe201a9a426396f Mon Sep 17 00:00:00 2001 From: karimtera Date: Thu, 17 Nov 2022 00:09:00 +0200 Subject: [PATCH 1/2] [WIP] adding exit code to the preprocessor --- verilog/preprocessor/verilog_preprocess.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/verilog/preprocessor/verilog_preprocess.h b/verilog/preprocessor/verilog_preprocess.h index 69d975983..d58135736 100644 --- a/verilog/preprocessor/verilog_preprocess.h +++ b/verilog/preprocessor/verilog_preprocess.h @@ -41,6 +41,7 @@ #include #include #include +#include #include "absl/status/status.h" #include "absl/status/statusor.h" @@ -87,6 +88,10 @@ struct VerilogPreprocessData { // are two separate vectors. std::vector errors; std::vector warnings; + + // An exit code of 4 bits is used to distinguish between errors met during + // preprocessing. + std::bitset<4> exit_code; }; // VerilogPreprocess transforms a TokenStreamView. From 5662702de0bfb8d149259891f1f84f80c0fa8d35 Mon Sep 17 00:00:00 2001 From: karimtera Date: Fri, 18 Nov 2022 19:55:41 +0200 Subject: [PATCH 2/2] [WIP] implementing exit codes into VerilogPreprocess --- verilog/preprocessor/verilog_preprocess.cc | 75 ++++++++++++++----- verilog/preprocessor/verilog_preprocess.h | 34 ++++++--- .../preprocessor/verilog_preprocessor.cc | 7 +- 3 files changed, 83 insertions(+), 33 deletions(-) diff --git a/verilog/preprocessor/verilog_preprocess.cc b/verilog/preprocessor/verilog_preprocess.cc index 93724bd90..115be84ec 100644 --- a/verilog/preprocessor/verilog_preprocess.cc +++ b/verilog/preprocessor/verilog_preprocess.cc @@ -59,11 +59,14 @@ VerilogPreprocess::VerilogPreprocess(const Config& config, FileOpener opener) } TokenStreamView::const_iterator VerilogPreprocess::GenerateBypassWhiteSpaces( - const StreamIteratorGenerator& generator) { - auto iterator = - generator(); // iterator should be pointing to a non-whitespace token; + const StreamIteratorGenerator& generator, + std::vector* backup_view = nullptr) { + // iterator should be pointing to a non-whitespace token; + auto iterator = generator(); + if (backup_view != nullptr) backup_view->push_back(iterator); while (verilog::VerilogLexer::KeepSyntaxTreeTokens(**iterator) == 0) { iterator = generator(); + if (backup_view != nullptr) backup_view->push_back(iterator); } return iterator; } @@ -223,7 +226,8 @@ std::unique_ptr VerilogPreprocess::ParseMacroDefinition( absl::Status VerilogPreprocess::ConsumeAndParseMacroCall( TokenStreamView::const_iterator iter, const StreamIteratorGenerator& generator, verible::MacroCall* macro_call, - const verible::MacroDefinition& macro_definition) { + const verible::MacroDefinition& macro_definition, + std::vector& backup_view) { // Parsing the macro . const absl::string_view macro_name_str = (*iter)->text().substr(1); verible::TokenInfo macro_name_token(MacroCallId, macro_name_str); @@ -238,10 +242,11 @@ absl::Status VerilogPreprocess::ConsumeAndParseMacroCall( // Parsing parameters. TokenStreamView::const_iterator token_iter = - GenerateBypassWhiteSpaces(generator); + GenerateBypassWhiteSpaces(generator, &backup_view); int parameters_size = macro_definition.Parameters().size(); if ((*token_iter)->text() == "(") { - token_iter = GenerateBypassWhiteSpaces(generator); // skip the "(" + token_iter = + GenerateBypassWhiteSpaces(generator, &backup_view); // skip the "(" } else { return absl::InvalidArgumentError( "Error it is illegal to call a callable macro without ()."); @@ -250,15 +255,15 @@ absl::Status VerilogPreprocess::ConsumeAndParseMacroCall( while (parameters_size > 0) { if ((*token_iter)->token_enum() == MacroArg) { macro_call->positional_arguments.emplace_back(**token_iter); - token_iter = GenerateBypassWhiteSpaces(generator); + token_iter = GenerateBypassWhiteSpaces(generator, &backup_view); if ((*token_iter)->text() == ",") - token_iter = GenerateBypassWhiteSpaces(generator); + token_iter = GenerateBypassWhiteSpaces(generator, &backup_view); parameters_size--; continue; } else if ((*token_iter)->text() == ",") { macro_call->positional_arguments.emplace_back( verible::DefaultTokenInfo()); - token_iter = GenerateBypassWhiteSpaces(generator); + token_iter = GenerateBypassWhiteSpaces(generator, &backup_view); parameters_size--; continue; } else if ((*token_iter)->text() == ")") @@ -287,6 +292,12 @@ absl::Status VerilogPreprocess::HandleMacroIdentifier( const auto* found = FindOrNull(preprocess_data_.macro_definitions, sv.substr(1)); if (!found) { + // Outputs the macro call if it couldn't be expanded. + preprocess_data_.preprocessed_token_stream.push_back(*iter); + + // Updates the exit code. + preprocess_data_.exit_code |= kMacroUseNotExpandedBit; + preprocess_data_.errors.push_back(VerilogPreprocessError( **iter, "Error expanding macro identifier, might not be defined before.")); @@ -294,15 +305,27 @@ absl::Status VerilogPreprocess::HandleMacroIdentifier( "Error expanding macro identifier, might not be defined before."); } - if (config_.expand_macros) { - verible::MacroCall macro_call; - if (auto status = - ConsumeAndParseMacroCall(iter, generator, ¯o_call, *found); - !status.ok()) - return status; - if (auto status = ExpandMacro(macro_call, found); !status.ok()) - return status; + // Backup of const_iterators to push back into + // preprocessor_data_.preprocessed_token_stream in case we couldn't open the + // file. + std::vector backup_view; + backup_view.push_back(iter); + + verible::MacroCall macro_call; + if (auto status = ConsumeAndParseMacroCall(iter, generator, ¯o_call, + *found, backup_view); + !status.ok()) { + // Outputs the macro call if it couldn't be expanded. + for (const auto& u : backup_view) + preprocess_data_.preprocessed_token_stream.push_back(*u); + + // Updates the exit code. + preprocess_data_.exit_code |= kMacroUseNotExpandedBit; + + return status; } + if (auto status = ExpandMacro(macro_call, found); !status.ok()) return status; + auto& lexed = preprocess_data_.lexed_macros_backup.back(); if (!forward) return absl::OkStatus(); auto iter_generator = verible::MakeConstIteratorStreamer(lexed); @@ -592,11 +615,17 @@ absl::Status VerilogPreprocess::HandleInclude( if (!file_opener_) return absl::FailedPreconditionError("file_opener_ is not defined"); + // Backup of const_iterators to push back into + // preprocessor_data_.preprocessed_token_stream in case we couldn't open the + // file. + std::vector backup_view; + backup_view.push_back(iter); + // TODO(karimtera): Support inclduing , // which should look for files defined by language standard in a compiler // dependent path. TokenStreamView::const_iterator token_iter = - GenerateBypassWhiteSpaces(generator); + GenerateBypassWhiteSpaces(generator, &backup_view); auto file_token_iter = *token_iter; if (file_token_iter->token_enum() != TK_StringLiteral) { preprocess_data_.errors.push_back( @@ -614,6 +643,14 @@ absl::Status VerilogPreprocess::HandleInclude( if (!status_or_file.ok()) { preprocess_data_.errors.push_back( {**token_iter, std::string(status_or_file.status().message())}); + // Outputs the `include directive as it is, in case we couldn't open the + // file. + for (const auto& u : backup_view) + preprocess_data_.preprocessed_token_stream.push_back(*u); + + // Add this error to the exit code. + preprocess_data_.exit_code |= kIncludeFileNotFoundBit; + return status_or_file.status(); } const absl::string_view source_contents = *status_or_file; @@ -740,7 +777,7 @@ VerilogPreprocessData VerilogPreprocess::ScanStream( const auto status = HandleTokenIterator(iter, iter_generator); if (!status.ok()) { // Detailed errors are already in preprocessor_data_.errors. - break; // For now, stop after first error. + /* break; // For now, stop after first error. */ } } diff --git a/verilog/preprocessor/verilog_preprocess.h b/verilog/preprocessor/verilog_preprocess.h index d58135736..e1361e137 100644 --- a/verilog/preprocessor/verilog_preprocess.h +++ b/verilog/preprocessor/verilog_preprocess.h @@ -7,10 +7,10 @@ // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. // VerilogPreprocess is a *pseudo*-preprocessor for Verilog. // Unlike a conventional preprocessor, this pseudo-preprocessor does not open @@ -35,13 +35,13 @@ #ifndef VERIBLE_VERILOG_PREPROCESSOR_VERILOG_PREPROCESS_H_ #define VERIBLE_VERILOG_PREPROCESSOR_VERILOG_PREPROCESS_H_ +#include #include #include #include #include #include #include -#include #include "absl/status/status.h" #include "absl/status/statusor.h" @@ -89,9 +89,9 @@ struct VerilogPreprocessData { std::vector errors; std::vector warnings; - // An exit code of 4 bits is used to distinguish between errors met during + // An exit code of 4 bits is used to distinguish between errors met during // preprocessing. - std::bitset<4> exit_code; + std::uint32_t exit_code{0}; }; // VerilogPreprocess transforms a TokenStreamView. @@ -169,10 +169,10 @@ class VerilogPreprocess { absl::Status HandleElse(TokenStreamView::const_iterator else_pos); absl::Status HandleEndif(TokenStreamView::const_iterator endif_pos); - static absl::Status ConsumeAndParseMacroCall(TokenStreamView::const_iterator, - const StreamIteratorGenerator&, - verible::MacroCall*, - const verible::MacroDefinition&); + static absl::Status ConsumeAndParseMacroCall( + TokenStreamView::const_iterator, const StreamIteratorGenerator&, + verible::MacroCall*, const verible::MacroDefinition&, + std::vector&); // The following functions return nullptr when there is no error: absl::Status ConsumeMacroDefinition(const StreamIteratorGenerator&, @@ -193,7 +193,8 @@ class VerilogPreprocess { // Generate a const_iterator to a non-whitespace token. static TokenStreamView::const_iterator GenerateBypassWhiteSpaces( - const StreamIteratorGenerator&); + const StreamIteratorGenerator&, + std::vector*); const Config config_; @@ -258,6 +259,15 @@ class VerilogPreprocess { // A pointer to a file opener function. // This is needed for opening new files while handling includes. const FileOpener file_opener_ = nullptr; + + enum ResultBits { + // file not found or tokenization error + kFileErrorBit = (1 << 0), + // At least one macro could not be expanded + kMacroUseNotExpandedBit = (1 << 1), + // At least one include file could not be found + kIncludeFileNotFoundBit = (1 << 2) + }; }; } // namespace verilog diff --git a/verilog/tools/preprocessor/verilog_preprocessor.cc b/verilog/tools/preprocessor/verilog_preprocessor.cc index aebc4d00a..379bc921c 100644 --- a/verilog/tools/preprocessor/verilog_preprocessor.cc +++ b/verilog/tools/preprocessor/verilog_preprocessor.cc @@ -124,9 +124,12 @@ static absl::Status PreprocessSingleFile( preprocessor.ScanStream(lexed_streamview); auto& preprocessed_stream = preprocessed_data.preprocessed_token_stream; for (auto u : preprocessed_stream) outs << u->text(); - for (auto& u : preprocessed_data.errors) outs << u.error_message << '\n'; - if (!preprocessed_data.errors.empty()) + for (auto& u : preprocessed_data.errors) + message_stream << u.error_message << '\n'; + if (!preprocessed_data.errors.empty()) { + message_stream << "Exit code: " << preprocessed_data.exit_code << "\n"; return absl::InvalidArgumentError("Error: The preprocessing has failed."); + } return absl::OkStatus(); }