Bug 256433

please hunt down the exact macros that trigger this behavior and supply a test case which does not #include anything.

commit 81fd61a940fb0e2261c54132d5c0a11b4a576039
branch master
Author: Milian Wolff <mail@milianw.de>
Date:   Sat Nov 27 19:31:29 2010 +0100

    Fix C++ preprocessor implementation to work according to the standard
    
    This fixes Bug 256433. Also some other macros from Boost library are now
    expanded correctly.
    
    BUG: 256433

diff --git a/languages/cpp/parser/rpp/pp-engine.cpp b/languages/cpp/parser/rpp/pp-engine.cpp
index b5b70f7..0336a03 100644
--- a/languages/cpp/parser/rpp/pp-engine.cpp
+++ b/languages/cpp/parser/rpp/pp-engine.cpp
@@ -155,7 +155,8 @@ void pp::handle_directive(uint directive, Stream& input, Stream& output)
 
 void pp::handle_include(bool skip_current_path, Stream& input, Stream& output)
 {
-  if (isLetter(input.current()) || input == '_') {
+  QByteArray bytes = KDevelop::IndexedString::fromIndex(input.current()).byteArray();
+  if (bytes.size() > 0 && (isLetter(bytes.at(0)) || bytes.at(0) == '_')) {
     pp_macro_expander expand_include(this);
 
     Anchor inputPosition = input.inputPosition();
@@ -199,7 +200,6 @@ void pp::handle_include(bool skip_current_path, Stream& input, Stream& output)
   while (!input.atEnd() && input != quote) {
     RETURN_ON_FAIL(input != '\n');
 
-    if(((uint)input) != indexFromCharacter(' '))
       includeNameB.append(input);
     ++input;
   }
@@ -528,6 +528,10 @@ Value pp::eval_primary(Stream& input)
       result.set_long(eval_primary(input).is_zero());
       break;
 
+    case '~':
+      result.set_long(~ eval_primary(input).l);
+      break;
+
     case '(':
       result = eval_constant_expression(input);
       token = next_token(input);
diff --git a/languages/cpp/parser/rpp/pp-macro-expander.cpp b/languages/cpp/parser/rpp/pp-macro-expander.cpp
index ba5916d..c664d87 100644
--- a/languages/cpp/parser/rpp/pp-macro-expander.cpp
+++ b/languages/cpp/parser/rpp/pp-macro-expander.cpp
@@ -38,7 +38,7 @@
 #include "preprocessor.h"
 #include "chartools.h"
 
-const int maxMacroExpansionDepth = 50;
+const int maxMacroExpansionDepth = 70;
 
 using namespace KDevelop;
 
@@ -183,7 +183,7 @@ class MacroHider {
     Environment* m_environment;
 };
 
-void pp_macro_expander::operator()(Stream& input, Stream& output)
+void pp_macro_expander::operator()(Stream& input, Stream& output, bool substitute, LocationTable* table)
 {
   skip_blanks(input, output);
 
@@ -214,47 +214,25 @@ void pp_macro_expander::operator()(Stream& input, Stream& output)
           ++input;
           skip_blanks (input, devnull());
           
+          // Need to extract previous identifier in case there are spaces in front of current output position
+          // May happen if parameter to the left of ## was expanded to an empty string
           IndexedString previous;
           if (output.offset() > 0) {
             previous = IndexedString::fromIndex(output.popLastOutput()); //Previous already has been expanded
-            while(output.offset() > 0 && isCharacter(previous.index()) && characterFromIndex(previous.index()) == ' ')
+            while(output.offset() > 0 && isCharacter(previous.index()) && isSpace(characterFromIndex(previous.index())))
               previous = IndexedString::fromIndex(output.popLastOutput());   
           }
-          
-          IndexedString add = IndexedString::fromIndex(skip_identifier (input));
-          
-          PreprocessedContents newExpanded;
-          
-          {
-            //Expand "add", so it is eventually replaced by an actual
-            PreprocessedContents actualText;
-            actualText.append(add.index());
-
-            {
-              Stream as(&actualText);
-              pp_macro_expander expand_actual(m_engine, m_frame);
-              Stream nas(&newExpanded);
-              expand_actual(as, nas);
-            }
-          }
-          
-          if(!newExpanded.isEmpty()) {
-            IndexedString first = IndexedString::fromIndex(newExpanded.first());
-            if(!isCharacter(first.index()) || QChar(characterFromIndex(first.index())).isLetterOrNumber() || characterFromIndex(first.index()) == '_') {
-              //Merge the tokens
-              newExpanded.first() = IndexedString(previous.byteArray() + first.byteArray()).index();
-            }else{
-              //Cannot merge, prepend the previous text
-              newExpanded.prepend(previous.index());
-            }
+          output.appendString(output.currentOutputAnchor(), previous);
+          // OK to put the merged tokens into stream separately, because the stream in character-based
+          Anchor nextStart = input.inputPosition();
+          IndexedString next = IndexedString::fromIndex(skip_identifier (input));
+          pp_actual actualNext = resolve_formal(next, input);
+          if (!actualNext.isValid()) {
+            output.appendString(nextStart, next);
           }else{
-            newExpanded.append(previous.index());
+            output.appendString(actualNext.sourcePosition, actualNext.sourceText);
           }
-          
-          //Now expand the merged text completely into the output.
-          pp_macro_expander final_expansion(m_engine, m_frame);
-          Stream nas(&newExpanded, output.currentOutputAnchor());
-          final_expansion(nas, output);
+          output.mark(input.inputPosition());
           continue;
         }
         
@@ -264,7 +242,7 @@ void pp_macro_expander::operator()(Stream& input, Stream& output)
 
         Anchor inputPosition = input.inputPosition();
         KDevelop::CursorInRevision originalInputPosition = input.originalInputPosition();
-        PreprocessedContents formal = resolve_formal(identifier, input).mergeText();
+        PreprocessedContents formal = resolve_formal(identifier, input).sourceText;
         
         //Escape so we don't break on '"'
         for(int a = formal.count()-1; a >= 0; --a) {
@@ -277,7 +255,6 @@ void pp_macro_expander::operator()(Stream& input, Stream& output)
             }
               
         }
-
         Stream is(&formal, inputPosition);
         is.setOriginalInputPosition(originalInputPosition);
         skip_whitespaces(is, devnull());
@@ -325,9 +302,34 @@ void pp_macro_expander::operator()(Stream& input, Stream& output)
         check_header_section
         
         Anchor inputPosition = input.inputPosition();
+        int offset = input.offset();
         IndexedString name = IndexedString::fromIndex(skip_identifier (input));
         
-        Anchor inputPosition2 = input.inputPosition();
+        // peek forward to check for ##
+        int start = input.offset();
+        skip_blanks(input, devnull());
+        if (!input.atEnd()) {
+          if(input == '#' && (++input) == '#') {
+              ++input;
+              //We have skipped a paste token
+              skip_blanks(input, devnull());
+              pp_actual actualFirst = resolve_formal(name, input);
+
+              if (!actualFirst.isValid()) {
+                output.appendString(inputPosition, name);
+              } else {
+                output.appendString(actualFirst.sourcePosition, actualFirst.sourceText);
+              }
+
+              input.seek(start); // will need to process the second argument too
+              output.mark(input.inputPosition());
+              continue;
+          }else{
+            input.seek(start);
+          }
+        }
+
+        if (substitute) {
         pp_actual actual = resolve_formal(name, input);
         if (actual.isValid()) {
           Q_ASSERT(actual.text.size() == actual.inputPosition.size());
@@ -340,23 +342,9 @@ void pp_macro_expander::operator()(Stream& input, Stream& output)
             output.appendString(*cursorIt, *textIt);
           }
           output << ' '; //Insert a whitespace to omit implicit token merging
-          output.mark(input.inputPosition());
-          
-          if(actual.text.isEmpty()) {
-            int start = input.offset();
             
-            skip_blanks(input, devnull());
-            if (input.atEnd()) {
-              // this might happen in e.g. /usr/include/bits/mathcalls.h
-              continue;
-            }
-            //Omit paste tokens behind empty used actuals, else we will merge with the previous text
-            if(input == '#' && (++input) == '#') {
-              ++input;
-              //We have skipped a paste token
             }else{
-              input.seek(start);
-            }
+            output.appendString(inputPosition, name);
           }
           
           continue;
@@ -384,15 +372,29 @@ void pp_macro_expander::operator()(Stream& input, Stream& output)
             output.appendString(inputPosition, convertFromByteArray(QDate::currentDate().toString("\"MMM dd yyyy\"").toUtf8()));
           else if (name == timeIndex)
             output.appendString(inputPosition, convertFromByteArray(QTime::currentTime().toString("\"hh:mm:ss\"").toUtf8()));
-          else
+          else {
+            if (table) {
+              // In case of a merged token, find some borders for it inside a macro invocation
+              Anchor leftmost = table->positionAt(offset, *input.source(), true).first;
+              Anchor rightmost = table->positionAt(input.offset(), *input.source(), true).first;
+              // The order of parameters inside macro body may be different from its declaration
+              if (rightmost < leftmost) {
+                qSwap(rightmost, leftmost);
+              }
+              output.appendString(leftmost, name);
+              if (rightmost != leftmost) {
+                output.mark(rightmost);
+              }
+            } else {
             output.appendString(inputPosition, name);
+            }
+          }
           continue;
         }
         
         EnableMacroExpansion enable(output, input.inputPosition()); //Configure the output-stream so it marks all stored input-positions as transformed through a macro
 
           if (macro->definitionSize()) {
-            
             //Hide the expanded macro to prevent endless expansion
             MacroHider hideMacro(macro, m_engine->environment());
             
@@ -417,19 +419,17 @@ void pp_macro_expander::operator()(Stream& input, Stream& output)
               output << ' '; //Prevent implicit token merging
             }
           }
-        }else if(input == '(') {
+        }else if(input == '(' && !substitute) {
 
         //Eventually execute a function-macro
           
         IndexedString previous = IndexedString::fromIndex(indexFromCharacter(' ')); //Previous already has been expanded
         uint stepsBack = 0;
-        while(isCharacter(previous.index()) && characterFromIndex(previous.index()) == ' ' && output.peekLastOutput(stepsBack)) {
+        while(isCharacter(previous.index()) && isSpace(characterFromIndex(previous.index())) && output.peekLastOutput(stepsBack)) {
           previous = IndexedString::fromIndex(output.peekLastOutput(stepsBack));
           ++stepsBack;
         }
-        
         pp_macro* macro = m_engine->environment()->retrieveMacro(previous, false);
-        
         if(!macro || !macro->function_like || !macro->defined || macro->hidden) {
           output << input;
           ++input;
@@ -446,69 +446,14 @@ void pp_macro_expander::operator()(Stream& input, Stream& output)
         RETURN_IF_INPUT_BROKEN
 
         pp_macro_expander expand_actual(m_engine, m_frame);
+        skip_actual_parameter(input, *macro, actuals, expand_actual);
 
-        {
-          PreprocessedContents actualText;
-          skip_whitespaces(input, devnull());
-          Anchor actualStart = input.inputPosition();
-          {
-            Stream as(&actualText);
-            skip_argument_variadics(actuals, macro, input, as);
-          }
-          trim(actualText);
-
-          pp_actual newActual;
-          {
-            PreprocessedContents newActualText;
-            Stream as(&actualText, actualStart);
-            as.setOriginalInputPosition(input.originalInputPosition());
-
-            rpp::LocationTable table;
-            table.anchor(0, actualStart, 0);
-            Stream nas(&newActualText, actualStart, &table);
-            expand_actual(as, nas);
-            
-            table.splitByAnchors(newActualText, actualStart, newActual.text, newActual.inputPosition);
-          }
-          newActual.forceValid = true;
-          
-          actuals.append(newActual);
-        }
-
-        // TODO: why separate from the above?
         while (!input.atEnd() && input == ',')
         {
           ++input; // skip ','
           
           RETURN_IF_INPUT_BROKEN
-
-          {
-            PreprocessedContents actualText;
-            skip_whitespaces(input, devnull());
-            Anchor actualStart = input.inputPosition();
-            {
-              Stream as(&actualText);
-              skip_argument_variadics(actuals, macro, input, as);
-            }
-            trim(actualText);
-
-            pp_actual newActual;
-            {
-              PreprocessedContents newActualText;
-              Stream as(&actualText, actualStart);
-              as.setOriginalInputPosition(input.originalInputPosition());
-
-              PreprocessedContents actualText;
-              rpp::LocationTable table;
-              table.anchor(0, actualStart, 0);
-              Stream nas(&newActualText, actualStart, &table);
-              expand_actual(as, nas);
-              
-              table.splitByAnchors(newActualText, actualStart, newActual.text, newActual.inputPosition);
-            }
-            newActual.forceValid = true;
-            actuals.append(newActual);
-          }
+          skip_actual_parameter(input, *macro, actuals, expand_actual);
         }
 
         if( input != ')' ) {
@@ -559,8 +504,15 @@ void pp_macro_expander::operator()(Stream& input, Stream& output)
           
           ///@todo UGLY conversion
           Stream ms((uint*)macro->definition(), macro->definitionSize(), Anchor(input.inputPosition(), true));
-          ms.setOriginalInputPosition(input.originalInputPosition());
-          expand_macro(ms, output);
+
+          PreprocessedContents expansion_text;
+          rpp::LocationTable table;
+          Stream expansion_stream(&expansion_text, Anchor(input.inputPosition(), true), &table);
+          expand_macro(ms, expansion_stream, true);
+
+          Stream ns(&expansion_text, Anchor(input.inputPosition(), true));
+          ns.setOriginalInputPosition(input.originalInputPosition());
+          expand_macro(ns, output, false, &table);
           output << ' '; //Prevent implicit token merging
         }
       } else {
@@ -572,6 +524,36 @@ void pp_macro_expander::operator()(Stream& input, Stream& output)
   }
 }
 
+void pp_macro_expander::skip_actual_parameter(Stream& input, rpp::pp_macro& macro, QList< pp_actual >& actuals, pp_macro_expander& expander)
+{
+  PreprocessedContents actualText;
+  skip_whitespaces(input, devnull());
+  Anchor actualStart = input.inputPosition();
+  {
+    Stream as(&actualText);
+    skip_argument_variadics(actuals, &macro, input, as);
+  }
+  trim(actualText);
+
+  pp_actual newActual;
+  newActual.sourceText = actualText;
+  newActual.sourcePosition = actualStart;
+  {
+    PreprocessedContents newActualText;
+    Stream as(&actualText, actualStart);
+    as.setOriginalInputPosition(input.originalInputPosition());
+
+    rpp::LocationTable table;
+    table.anchor(0, actualStart, 0);
+    Stream nas(&newActualText, actualStart, &table);
+    expander(as, nas);
+    table.splitByAnchors(newActualText, actualStart, newActual.text, newActual.inputPosition);
+  }
+  newActual.forceValid = true;
+
+  actuals.append(newActual);
+}
+
 void pp_macro_expander::skip_argument_variadics (const QList<pp_actual>& __actuals, pp_macro *__macro, Stream& input, Stream& output)
 {
   int first;
@@ -586,4 +568,3 @@ void pp_macro_expander::skip_argument_variadics (const QList<pp_actual>& __actua
             && input == '.'
             && (__actuals.size() + 1) == (int)__macro->formalsSize());
 }
-
diff --git a/languages/cpp/parser/rpp/pp-macro-expander.h b/languages/cpp/parser/rpp/pp-macro-expander.h
index 2951c93..45716f7 100644
--- a/languages/cpp/parser/rpp/pp-macro-expander.h
+++ b/languages/cpp/parser/rpp/pp-macro-expander.h
@@ -45,6 +45,8 @@ class pp_actual {
 public:
   pp_actual() : forceValid(false) {
   }
+  PreprocessedContents sourceText;
+  Anchor sourcePosition;
   QList<PreprocessedContents> text;
   QList<Anchor> inputPosition; //Each inputPosition marks the beginning of one item in the text list
   bool forceValid;
@@ -57,17 +59,6 @@ public:
     text.clear();
     inputPosition.clear();
   }
-
-  PreprocessedContents mergeText() const {
-    if(text.count() == 1)
-      return text.at(0);
-    
-    PreprocessedContents ret;
-    
-    foreach(const PreprocessedContents& t, text)
-      ret += t;
-    return ret;
-  }
 };
 
 class pp_frame
@@ -89,7 +80,8 @@ public:
 
   /// Expands text with the known macros. Continues until it finds a new text line
   /// beginning with #, at which point control is returned.
-  void operator()(Stream& input, Stream& output);
+  /// If substitute == true, perform only macro parameter substitution and # token processing
+  void operator()(Stream& input, Stream& output, bool substitute = false, LocationTable* table = 0);
 
   void skip_argument_variadics (const QList<pp_actual>& __actuals, pp_macro *__macro,
                                 Stream& input, Stream& output);
@@ -107,6 +99,10 @@ public:
   }
   
 private:
+  /// Read actual parameter of @ref macro value from @ref input and append it to @ref actuals
+  /// @ref expander is a reusable macro expander
+  void skip_actual_parameter(rpp::Stream& input, rpp::pp_macro& macro, QList< rpp::pp_actual >& actuals, rpp::pp_macro_expander& expander);
+
   pp* m_engine;
   pp_frame* m_frame;
 
diff --git a/languages/cpp/parser/rpp/pp-stream.h b/languages/cpp/parser/rpp/pp-stream.h
index 2c00f12..bf7fdb6 100644
--- a/languages/cpp/parser/rpp/pp-stream.h
+++ b/languages/cpp/parser/rpp/pp-stream.h
@@ -163,6 +163,9 @@ class KDEVCPPRPP_EXPORT Stream
     Stream & operator<< ( const Stream& input );
     Stream& appendString( const Anchor& inputPosition, const PreprocessedContents & string );
     Stream& appendString( const Anchor& inputPosition, KDevelop::IndexedString index );
+    const PreprocessedContents* source() const {
+      return m_string;
+    }
 
   private:
     Q_DISABLE_COPY(Stream)
diff --git a/languages/cpp/parser/tests/test_parser.cpp b/languages/cpp/parser/tests/test_parser.cpp
index 475e50c..c03ede3 100644
--- a/languages/cpp/parser/tests/test_parser.cpp
+++ b/languages/cpp/parser/tests/test_parser.cpp
@@ -440,19 +440,34 @@ private slots:
 
   void testPreprocessor() {
     rpp::Preprocessor preprocessor;
-    //QCOMPARE(preprocess("#define TEST (1L<<10)\nTEST").trimmed(), QString("(1L<<10)"));
+    QCOMPARE(preprocess("#define TEST (1L<<10)\nTEST").trimmed(), QString("(1L<<10)"));
+    QCOMPARE(preprocess("#define SELF OTHER\n#define OTHER SELF\nSELF").trimmed(), QString("SELF"));
     QCOMPARE(preprocess("#define TEST //Comment\nTEST 1").trimmed(), QString("1")); //Comments are not included in macros
     QCOMPARE(preprocess("#define TEST /*Comment\n*/\nTEST 1").trimmed(), QString("1")); //Comments are not included in macros
     QCOMPARE(preprocess("#define TEST_URL \"http://foobar.com\"\nTEST_URL").trimmed(), QString("\"http://foobar.com\""));
     QCOMPARE(preprocess("#define TEST_STR \"//\\\"//\"\nTEST_STR").trimmed(), QString("\"//\\\"//\""));
+    QCOMPARE(preprocess("#if ~1\n#define NUMBER 10\n#else\n#define NUMBER 20\n#endif\nNUMBER").trimmed(), QString("10"));
+    QCOMPARE(preprocess("#define MACRO(a, b) ab\nMACRO\n(aa, bb)").trimmed(), QString("ab"));
+    QCOMPARE(preprocess("#define MACRO(a, b) ab\nMACRO(aa,\n bb)").trimmed(), QString("ab"));
+  }
 
+  void testPreprocessorStringify() {
+    QCOMPARE(preprocess("#define STR(s) #s\n#define MACRO string\nSTR(MACRO)").trimmed(), QString("\"MACRO\""));
+    QCOMPARE(preprocess("#define STR(s) #s\n#define XSTR(s) STR(s)\n#define MACRO string\nXSTR(MACRO)").simplified(), QString("\"string\""));
   }
 
   void testStringConcatenation()
   {
     rpp::Preprocessor preprocessor;
     QCOMPARE(preprocess("Hello##You"), QString("HelloYou"));
+    QCOMPARE(preprocess("#define CONCAT(Var1, Var2) Var1##Var2\nCONCAT(var1, )").trimmed(), QString("var1"));
+    QCOMPARE(preprocess("#define CONCAT(Var1, Var2) Var1##Var2\nCONCAT(, var2)").trimmed(), QString("var2"));
     QCOMPARE(preprocess("#define CONCAT(Var1, Var2) Var1##Var2 Var2##Var1\nCONCAT(      Hello      ,      You     )").simplified(), QString("\nHelloYou YouHello").simplified());
+
+    QCOMPARE(preprocess("#define GLUE(a, b) a ## b\n#define HIGHLOW hello\nGLUE(HIGH, LOW)").trimmed(), QString("hello"));
+    QCOMPARE(preprocess("#define GLUE(a, b) a ## b\n#define HIGHLOW hello\n#define LOW LOW world\nGLUE(HIGH, LOW)").trimmed(), QString("hello"));
+    QCOMPARE(preprocess("#define GLUE(a, b) a ##b\n#define XGLUE(a, b) GLUE(a, b)\n#define HIGHLOW hello\n#define LOW LOW world\nXGLUE(HIGH, LOW)").simplified(), QString("hello world")); // TODO: simplified -> trimmed
+    QCOMPARE(preprocess("#define GLUE(a, b, c) k ## l ## m\nGLUE(a, b, c)").trimmed(), QString("klm"));
   }
 
   void testCondition()
diff --git a/languages/cpp/tests/test_cppcodecompletion.cpp b/languages/cpp/tests/test_cppcodecompletion.cpp
index 0386878..663dabe 100644
--- a/languages/cpp/tests/test_cppcodecompletion.cpp
+++ b/languages/cpp/tests/test_cppcodecompletion.cpp
@@ -1785,9 +1785,24 @@ void TestCppCodeCompletion::testMacroExpansionRanges() {
   DUChainWriteLocker l(DUChain::lock());
   TopDUContext* ctx = parse(test.toUtf8());
   QCOMPARE(ctx->localDeclarations().count(), 1);
-  kDebug() << ctx->localDeclarations()[0]->range().castToSimpleRange().textRange();
   QCOMPARE(ctx->localDeclarations()[0]->range().castToSimpleRange().textRange(), KTextEditor::Range(1, 5, 1, 11));
 }
+{
+  //The range of the merged declaration name should be within macro invocation even when the order of merging is different from the order of formal parameters
+  QString test("#define TEST(X, Y) class Y ## X {};\nTEST(Hello, World)\n");
+  DUChainWriteLocker l(DUChain::lock());
+  TopDUContext* ctx = parse(test.toUtf8());
+  QCOMPARE(ctx->localDeclarations().count(), 1);
+  QCOMPARE(ctx->localDeclarations()[0]->range().castToSimpleRange().textRange(), KTextEditor::Range(1, 12, 1, 18));
+}
+{
+  //The range of the merged declaration name should be collapsed if it does not start with a macro parameter
+  QString test("#define TEST(X) class Hallo ## X {};\nTEST(Class)\n");
+  DUChainWriteLocker l(DUChain::lock());
+  TopDUContext* ctx = parse(test.toUtf8());
+  QCOMPARE(ctx->localDeclarations().count(), 1);
+  QCOMPARE(ctx->localDeclarations()[0]->range().castToSimpleRange().textRange(), KTextEditor::Range(1, 11, 1, 11));
+}
 }
 
 void TestCppCodeCompletion::testTimeMacro()

Moving all the bugs from the CPP Parser. It was not well defined the difference between it and C++ Language Support and people kept reporting in both places indistinctively