Version: SVN (using KDE 4.5.1) OS: Linux The following code #include <boost/test/unit_test.hpp> BOOST_AUTO_TEST_CASE (newtest) { BOOST_CHECK_EQUAL(10, 10); } reports an error with message "Call to macro f missing argument number 1" after BOOST_CHECK_EQUAL macro. However, this code is correct and compiles without errors. To compile the example you need the boost::test library installed (happens on boost 1.44). Reproducible: Always
please hunt down the exact macros that trigger this behavior and supply a test case which does not #include anything.
commit 81fd61a940fb0e2261c54132d5c0a11b4a576039 branch master Author: Milian Wolff <mail@milianw.de> Date: Sat Nov 27 19:31:29 2010 +0100 Fix C++ preprocessor implementation to work according to the standard This fixes Bug 256433. Also some other macros from Boost library are now expanded correctly. BUG: 256433 diff --git a/languages/cpp/parser/rpp/pp-engine.cpp b/languages/cpp/parser/rpp/pp-engine.cpp index b5b70f7..0336a03 100644 --- a/languages/cpp/parser/rpp/pp-engine.cpp +++ b/languages/cpp/parser/rpp/pp-engine.cpp @@ -155,7 +155,8 @@ void pp::handle_directive(uint directive, Stream& input, Stream& output) void pp::handle_include(bool skip_current_path, Stream& input, Stream& output) { - if (isLetter(input.current()) || input == '_') { + QByteArray bytes = KDevelop::IndexedString::fromIndex(input.current()).byteArray(); + if (bytes.size() > 0 && (isLetter(bytes.at(0)) || bytes.at(0) == '_')) { pp_macro_expander expand_include(this); Anchor inputPosition = input.inputPosition(); @@ -199,7 +200,6 @@ void pp::handle_include(bool skip_current_path, Stream& input, Stream& output) while (!input.atEnd() && input != quote) { RETURN_ON_FAIL(input != '\n'); - if(((uint)input) != indexFromCharacter(' ')) includeNameB.append(input); ++input; } @@ -528,6 +528,10 @@ Value pp::eval_primary(Stream& input) result.set_long(eval_primary(input).is_zero()); break; + case '~': + result.set_long(~ eval_primary(input).l); + break; + case '(': result = eval_constant_expression(input); token = next_token(input); diff --git a/languages/cpp/parser/rpp/pp-macro-expander.cpp b/languages/cpp/parser/rpp/pp-macro-expander.cpp index ba5916d..c664d87 100644 --- a/languages/cpp/parser/rpp/pp-macro-expander.cpp +++ b/languages/cpp/parser/rpp/pp-macro-expander.cpp @@ -38,7 +38,7 @@ #include "preprocessor.h" #include "chartools.h" -const int maxMacroExpansionDepth = 50; +const int maxMacroExpansionDepth = 70; using namespace KDevelop; @@ -183,7 +183,7 @@ class MacroHider { Environment* m_environment; }; -void pp_macro_expander::operator()(Stream& input, Stream& output) +void pp_macro_expander::operator()(Stream& input, Stream& output, bool substitute, LocationTable* table) { skip_blanks(input, output); @@ -214,47 +214,25 @@ void pp_macro_expander::operator()(Stream& input, Stream& output) ++input; skip_blanks (input, devnull()); + // Need to extract previous identifier in case there are spaces in front of current output position + // May happen if parameter to the left of ## was expanded to an empty string IndexedString previous; if (output.offset() > 0) { previous = IndexedString::fromIndex(output.popLastOutput()); //Previous already has been expanded - while(output.offset() > 0 && isCharacter(previous.index()) && characterFromIndex(previous.index()) == ' ') + while(output.offset() > 0 && isCharacter(previous.index()) && isSpace(characterFromIndex(previous.index()))) previous = IndexedString::fromIndex(output.popLastOutput()); } - - IndexedString add = IndexedString::fromIndex(skip_identifier (input)); - - PreprocessedContents newExpanded; - - { - //Expand "add", so it is eventually replaced by an actual - PreprocessedContents actualText; - actualText.append(add.index()); - - { - Stream as(&actualText); - pp_macro_expander expand_actual(m_engine, m_frame); - Stream nas(&newExpanded); - expand_actual(as, nas); - } - } - - if(!newExpanded.isEmpty()) { - IndexedString first = IndexedString::fromIndex(newExpanded.first()); - if(!isCharacter(first.index()) || QChar(characterFromIndex(first.index())).isLetterOrNumber() || characterFromIndex(first.index()) == '_') { - //Merge the tokens - newExpanded.first() = IndexedString(previous.byteArray() + first.byteArray()).index(); - }else{ - //Cannot merge, prepend the previous text - newExpanded.prepend(previous.index()); - } + output.appendString(output.currentOutputAnchor(), previous); + // OK to put the merged tokens into stream separately, because the stream in character-based + Anchor nextStart = input.inputPosition(); + IndexedString next = IndexedString::fromIndex(skip_identifier (input)); + pp_actual actualNext = resolve_formal(next, input); + if (!actualNext.isValid()) { + output.appendString(nextStart, next); }else{ - newExpanded.append(previous.index()); + output.appendString(actualNext.sourcePosition, actualNext.sourceText); } - - //Now expand the merged text completely into the output. - pp_macro_expander final_expansion(m_engine, m_frame); - Stream nas(&newExpanded, output.currentOutputAnchor()); - final_expansion(nas, output); + output.mark(input.inputPosition()); continue; } @@ -264,7 +242,7 @@ void pp_macro_expander::operator()(Stream& input, Stream& output) Anchor inputPosition = input.inputPosition(); KDevelop::CursorInRevision originalInputPosition = input.originalInputPosition(); - PreprocessedContents formal = resolve_formal(identifier, input).mergeText(); + PreprocessedContents formal = resolve_formal(identifier, input).sourceText; //Escape so we don't break on '"' for(int a = formal.count()-1; a >= 0; --a) { @@ -277,7 +255,6 @@ void pp_macro_expander::operator()(Stream& input, Stream& output) } } - Stream is(&formal, inputPosition); is.setOriginalInputPosition(originalInputPosition); skip_whitespaces(is, devnull()); @@ -325,9 +302,34 @@ void pp_macro_expander::operator()(Stream& input, Stream& output) check_header_section Anchor inputPosition = input.inputPosition(); + int offset = input.offset(); IndexedString name = IndexedString::fromIndex(skip_identifier (input)); - Anchor inputPosition2 = input.inputPosition(); + // peek forward to check for ## + int start = input.offset(); + skip_blanks(input, devnull()); + if (!input.atEnd()) { + if(input == '#' && (++input) == '#') { + ++input; + //We have skipped a paste token + skip_blanks(input, devnull()); + pp_actual actualFirst = resolve_formal(name, input); + + if (!actualFirst.isValid()) { + output.appendString(inputPosition, name); + } else { + output.appendString(actualFirst.sourcePosition, actualFirst.sourceText); + } + + input.seek(start); // will need to process the second argument too + output.mark(input.inputPosition()); + continue; + }else{ + input.seek(start); + } + } + + if (substitute) { pp_actual actual = resolve_formal(name, input); if (actual.isValid()) { Q_ASSERT(actual.text.size() == actual.inputPosition.size()); @@ -340,23 +342,9 @@ void pp_macro_expander::operator()(Stream& input, Stream& output) output.appendString(*cursorIt, *textIt); } output << ' '; //Insert a whitespace to omit implicit token merging - output.mark(input.inputPosition()); - - if(actual.text.isEmpty()) { - int start = input.offset(); - skip_blanks(input, devnull()); - if (input.atEnd()) { - // this might happen in e.g. /usr/include/bits/mathcalls.h - continue; - } - //Omit paste tokens behind empty used actuals, else we will merge with the previous text - if(input == '#' && (++input) == '#') { - ++input; - //We have skipped a paste token }else{ - input.seek(start); - } + output.appendString(inputPosition, name); } continue; @@ -384,15 +372,29 @@ void pp_macro_expander::operator()(Stream& input, Stream& output) output.appendString(inputPosition, convertFromByteArray(QDate::currentDate().toString("\"MMM dd yyyy\"").toUtf8())); else if (name == timeIndex) output.appendString(inputPosition, convertFromByteArray(QTime::currentTime().toString("\"hh:mm:ss\"").toUtf8())); - else + else { + if (table) { + // In case of a merged token, find some borders for it inside a macro invocation + Anchor leftmost = table->positionAt(offset, *input.source(), true).first; + Anchor rightmost = table->positionAt(input.offset(), *input.source(), true).first; + // The order of parameters inside macro body may be different from its declaration + if (rightmost < leftmost) { + qSwap(rightmost, leftmost); + } + output.appendString(leftmost, name); + if (rightmost != leftmost) { + output.mark(rightmost); + } + } else { output.appendString(inputPosition, name); + } + } continue; } EnableMacroExpansion enable(output, input.inputPosition()); //Configure the output-stream so it marks all stored input-positions as transformed through a macro if (macro->definitionSize()) { - //Hide the expanded macro to prevent endless expansion MacroHider hideMacro(macro, m_engine->environment()); @@ -417,19 +419,17 @@ void pp_macro_expander::operator()(Stream& input, Stream& output) output << ' '; //Prevent implicit token merging } } - }else if(input == '(') { + }else if(input == '(' && !substitute) { //Eventually execute a function-macro IndexedString previous = IndexedString::fromIndex(indexFromCharacter(' ')); //Previous already has been expanded uint stepsBack = 0; - while(isCharacter(previous.index()) && characterFromIndex(previous.index()) == ' ' && output.peekLastOutput(stepsBack)) { + while(isCharacter(previous.index()) && isSpace(characterFromIndex(previous.index())) && output.peekLastOutput(stepsBack)) { previous = IndexedString::fromIndex(output.peekLastOutput(stepsBack)); ++stepsBack; } - pp_macro* macro = m_engine->environment()->retrieveMacro(previous, false); - if(!macro || !macro->function_like || !macro->defined || macro->hidden) { output << input; ++input; @@ -446,69 +446,14 @@ void pp_macro_expander::operator()(Stream& input, Stream& output) RETURN_IF_INPUT_BROKEN pp_macro_expander expand_actual(m_engine, m_frame); + skip_actual_parameter(input, *macro, actuals, expand_actual); - { - PreprocessedContents actualText; - skip_whitespaces(input, devnull()); - Anchor actualStart = input.inputPosition(); - { - Stream as(&actualText); - skip_argument_variadics(actuals, macro, input, as); - } - trim(actualText); - - pp_actual newActual; - { - PreprocessedContents newActualText; - Stream as(&actualText, actualStart); - as.setOriginalInputPosition(input.originalInputPosition()); - - rpp::LocationTable table; - table.anchor(0, actualStart, 0); - Stream nas(&newActualText, actualStart, &table); - expand_actual(as, nas); - - table.splitByAnchors(newActualText, actualStart, newActual.text, newActual.inputPosition); - } - newActual.forceValid = true; - - actuals.append(newActual); - } - - // TODO: why separate from the above? while (!input.atEnd() && input == ',') { ++input; // skip ',' RETURN_IF_INPUT_BROKEN - - { - PreprocessedContents actualText; - skip_whitespaces(input, devnull()); - Anchor actualStart = input.inputPosition(); - { - Stream as(&actualText); - skip_argument_variadics(actuals, macro, input, as); - } - trim(actualText); - - pp_actual newActual; - { - PreprocessedContents newActualText; - Stream as(&actualText, actualStart); - as.setOriginalInputPosition(input.originalInputPosition()); - - PreprocessedContents actualText; - rpp::LocationTable table; - table.anchor(0, actualStart, 0); - Stream nas(&newActualText, actualStart, &table); - expand_actual(as, nas); - - table.splitByAnchors(newActualText, actualStart, newActual.text, newActual.inputPosition); - } - newActual.forceValid = true; - actuals.append(newActual); - } + skip_actual_parameter(input, *macro, actuals, expand_actual); } if( input != ')' ) { @@ -559,8 +504,15 @@ void pp_macro_expander::operator()(Stream& input, Stream& output) ///@todo UGLY conversion Stream ms((uint*)macro->definition(), macro->definitionSize(), Anchor(input.inputPosition(), true)); - ms.setOriginalInputPosition(input.originalInputPosition()); - expand_macro(ms, output); + + PreprocessedContents expansion_text; + rpp::LocationTable table; + Stream expansion_stream(&expansion_text, Anchor(input.inputPosition(), true), &table); + expand_macro(ms, expansion_stream, true); + + Stream ns(&expansion_text, Anchor(input.inputPosition(), true)); + ns.setOriginalInputPosition(input.originalInputPosition()); + expand_macro(ns, output, false, &table); output << ' '; //Prevent implicit token merging } } else { @@ -572,6 +524,36 @@ void pp_macro_expander::operator()(Stream& input, Stream& output) } } +void pp_macro_expander::skip_actual_parameter(Stream& input, rpp::pp_macro& macro, QList< pp_actual >& actuals, pp_macro_expander& expander) +{ + PreprocessedContents actualText; + skip_whitespaces(input, devnull()); + Anchor actualStart = input.inputPosition(); + { + Stream as(&actualText); + skip_argument_variadics(actuals, ¯o, input, as); + } + trim(actualText); + + pp_actual newActual; + newActual.sourceText = actualText; + newActual.sourcePosition = actualStart; + { + PreprocessedContents newActualText; + Stream as(&actualText, actualStart); + as.setOriginalInputPosition(input.originalInputPosition()); + + rpp::LocationTable table; + table.anchor(0, actualStart, 0); + Stream nas(&newActualText, actualStart, &table); + expander(as, nas); + table.splitByAnchors(newActualText, actualStart, newActual.text, newActual.inputPosition); + } + newActual.forceValid = true; + + actuals.append(newActual); +} + void pp_macro_expander::skip_argument_variadics (const QList<pp_actual>& __actuals, pp_macro *__macro, Stream& input, Stream& output) { int first; @@ -586,4 +568,3 @@ void pp_macro_expander::skip_argument_variadics (const QList<pp_actual>& __actua && input == '.' && (__actuals.size() + 1) == (int)__macro->formalsSize()); } - diff --git a/languages/cpp/parser/rpp/pp-macro-expander.h b/languages/cpp/parser/rpp/pp-macro-expander.h index 2951c93..45716f7 100644 --- a/languages/cpp/parser/rpp/pp-macro-expander.h +++ b/languages/cpp/parser/rpp/pp-macro-expander.h @@ -45,6 +45,8 @@ class pp_actual { public: pp_actual() : forceValid(false) { } + PreprocessedContents sourceText; + Anchor sourcePosition; QList<PreprocessedContents> text; QList<Anchor> inputPosition; //Each inputPosition marks the beginning of one item in the text list bool forceValid; @@ -57,17 +59,6 @@ public: text.clear(); inputPosition.clear(); } - - PreprocessedContents mergeText() const { - if(text.count() == 1) - return text.at(0); - - PreprocessedContents ret; - - foreach(const PreprocessedContents& t, text) - ret += t; - return ret; - } }; class pp_frame @@ -89,7 +80,8 @@ public: /// Expands text with the known macros. Continues until it finds a new text line /// beginning with #, at which point control is returned. - void operator()(Stream& input, Stream& output); + /// If substitute == true, perform only macro parameter substitution and # token processing + void operator()(Stream& input, Stream& output, bool substitute = false, LocationTable* table = 0); void skip_argument_variadics (const QList<pp_actual>& __actuals, pp_macro *__macro, Stream& input, Stream& output); @@ -107,6 +99,10 @@ public: } private: + /// Read actual parameter of @ref macro value from @ref input and append it to @ref actuals + /// @ref expander is a reusable macro expander + void skip_actual_parameter(rpp::Stream& input, rpp::pp_macro& macro, QList< rpp::pp_actual >& actuals, rpp::pp_macro_expander& expander); + pp* m_engine; pp_frame* m_frame; diff --git a/languages/cpp/parser/rpp/pp-stream.h b/languages/cpp/parser/rpp/pp-stream.h index 2c00f12..bf7fdb6 100644 --- a/languages/cpp/parser/rpp/pp-stream.h +++ b/languages/cpp/parser/rpp/pp-stream.h @@ -163,6 +163,9 @@ class KDEVCPPRPP_EXPORT Stream Stream & operator<< ( const Stream& input ); Stream& appendString( const Anchor& inputPosition, const PreprocessedContents & string ); Stream& appendString( const Anchor& inputPosition, KDevelop::IndexedString index ); + const PreprocessedContents* source() const { + return m_string; + } private: Q_DISABLE_COPY(Stream) diff --git a/languages/cpp/parser/tests/test_parser.cpp b/languages/cpp/parser/tests/test_parser.cpp index 475e50c..c03ede3 100644 --- a/languages/cpp/parser/tests/test_parser.cpp +++ b/languages/cpp/parser/tests/test_parser.cpp @@ -440,19 +440,34 @@ private slots: void testPreprocessor() { rpp::Preprocessor preprocessor; - //QCOMPARE(preprocess("#define TEST (1L<<10)\nTEST").trimmed(), QString("(1L<<10)")); + QCOMPARE(preprocess("#define TEST (1L<<10)\nTEST").trimmed(), QString("(1L<<10)")); + QCOMPARE(preprocess("#define SELF OTHER\n#define OTHER SELF\nSELF").trimmed(), QString("SELF")); QCOMPARE(preprocess("#define TEST //Comment\nTEST 1").trimmed(), QString("1")); //Comments are not included in macros QCOMPARE(preprocess("#define TEST /*Comment\n*/\nTEST 1").trimmed(), QString("1")); //Comments are not included in macros QCOMPARE(preprocess("#define TEST_URL \"http://foobar.com\"\nTEST_URL").trimmed(), QString("\"http://foobar.com\"")); QCOMPARE(preprocess("#define TEST_STR \"//\\\"//\"\nTEST_STR").trimmed(), QString("\"//\\\"//\"")); + QCOMPARE(preprocess("#if ~1\n#define NUMBER 10\n#else\n#define NUMBER 20\n#endif\nNUMBER").trimmed(), QString("10")); + QCOMPARE(preprocess("#define MACRO(a, b) ab\nMACRO\n(aa, bb)").trimmed(), QString("ab")); + QCOMPARE(preprocess("#define MACRO(a, b) ab\nMACRO(aa,\n bb)").trimmed(), QString("ab")); + } + void testPreprocessorStringify() { + QCOMPARE(preprocess("#define STR(s) #s\n#define MACRO string\nSTR(MACRO)").trimmed(), QString("\"MACRO\"")); + QCOMPARE(preprocess("#define STR(s) #s\n#define XSTR(s) STR(s)\n#define MACRO string\nXSTR(MACRO)").simplified(), QString("\"string\"")); } void testStringConcatenation() { rpp::Preprocessor preprocessor; QCOMPARE(preprocess("Hello##You"), QString("HelloYou")); + QCOMPARE(preprocess("#define CONCAT(Var1, Var2) Var1##Var2\nCONCAT(var1, )").trimmed(), QString("var1")); + QCOMPARE(preprocess("#define CONCAT(Var1, Var2) Var1##Var2\nCONCAT(, var2)").trimmed(), QString("var2")); QCOMPARE(preprocess("#define CONCAT(Var1, Var2) Var1##Var2 Var2##Var1\nCONCAT( Hello , You )").simplified(), QString("\nHelloYou YouHello").simplified()); + + QCOMPARE(preprocess("#define GLUE(a, b) a ## b\n#define HIGHLOW hello\nGLUE(HIGH, LOW)").trimmed(), QString("hello")); + QCOMPARE(preprocess("#define GLUE(a, b) a ## b\n#define HIGHLOW hello\n#define LOW LOW world\nGLUE(HIGH, LOW)").trimmed(), QString("hello")); + QCOMPARE(preprocess("#define GLUE(a, b) a ##b\n#define XGLUE(a, b) GLUE(a, b)\n#define HIGHLOW hello\n#define LOW LOW world\nXGLUE(HIGH, LOW)").simplified(), QString("hello world")); // TODO: simplified -> trimmed + QCOMPARE(preprocess("#define GLUE(a, b, c) k ## l ## m\nGLUE(a, b, c)").trimmed(), QString("klm")); } void testCondition() diff --git a/languages/cpp/tests/test_cppcodecompletion.cpp b/languages/cpp/tests/test_cppcodecompletion.cpp index 0386878..663dabe 100644 --- a/languages/cpp/tests/test_cppcodecompletion.cpp +++ b/languages/cpp/tests/test_cppcodecompletion.cpp @@ -1785,9 +1785,24 @@ void TestCppCodeCompletion::testMacroExpansionRanges() { DUChainWriteLocker l(DUChain::lock()); TopDUContext* ctx = parse(test.toUtf8()); QCOMPARE(ctx->localDeclarations().count(), 1); - kDebug() << ctx->localDeclarations()[0]->range().castToSimpleRange().textRange(); QCOMPARE(ctx->localDeclarations()[0]->range().castToSimpleRange().textRange(), KTextEditor::Range(1, 5, 1, 11)); } +{ + //The range of the merged declaration name should be within macro invocation even when the order of merging is different from the order of formal parameters + QString test("#define TEST(X, Y) class Y ## X {};\nTEST(Hello, World)\n"); + DUChainWriteLocker l(DUChain::lock()); + TopDUContext* ctx = parse(test.toUtf8()); + QCOMPARE(ctx->localDeclarations().count(), 1); + QCOMPARE(ctx->localDeclarations()[0]->range().castToSimpleRange().textRange(), KTextEditor::Range(1, 12, 1, 18)); +} +{ + //The range of the merged declaration name should be collapsed if it does not start with a macro parameter + QString test("#define TEST(X) class Hallo ## X {};\nTEST(Class)\n"); + DUChainWriteLocker l(DUChain::lock()); + TopDUContext* ctx = parse(test.toUtf8()); + QCOMPARE(ctx->localDeclarations().count(), 1); + QCOMPARE(ctx->localDeclarations()[0]->range().castToSimpleRange().textRange(), KTextEditor::Range(1, 11, 1, 11)); +} } void TestCppCodeCompletion::testTimeMacro()
Moving all the bugs from the CPP Parser. It was not well defined the difference between it and C++ Language Support and people kept reporting in both places indistinctively