Doxygen
pyscanner.l
浏览该文件的文档.
1 /******************************************************************************
2  *
3  * Copyright (C) 1997-2021 by Dimitri van Heesch.
4  *
5  * Permission to use, copy, modify, and distribute this software and its
6  * documentation under the terms of the GNU General Public License is hereby
7  * granted. No representations are made about the suitability of this software
8  * for any purpose. It is provided "as is" without express or implied warranty.
9  * See the GNU General Public License for more details.
10  *
11  * Documents produced by Doxygen are derivative works derived from the
12  * input used in their production; they are not affected by this license.
13  *
14  */
15 /* This code is based on the work done by the MoxyPyDoxy team
16  * (Linda Leong, Mike Rivera, Kim Truong, and Gabriel Estrada)
17  * in Spring 2005 as part of CS 179E: Compiler Design Project
18  * at the University of California, Riverside; the course was
19  * taught by Peter H. Froehlich <phf@acm.org>.
20  */
21 
22 %option never-interactive
23 %option prefix="pyscannerYY"
24 %option reentrant
25 %option extra-type="struct pyscannerYY_state *"
26 %top{
27 #include <stdint.h>
28 // forward declare yyscan_t to improve type safety
29 #define YY_TYPEDEF_YY_SCANNER_T
30 struct yyguts_t;
31 typedef yyguts_t *yyscan_t;
32 }
33 
34 %{
35 
36 /*
37  * includes
38  */
39 #include <stdio.h>
40 #include <stdlib.h>
41 #include <assert.h>
42 #include <ctype.h>
43 
44 #include "pyscanner.h"
45 #include "entry.h"
46 #include "message.h"
47 #include "config.h"
48 #include "doxygen.h"
49 #include "util.h"
50 #include "defargs.h"
51 #include "language.h"
52 #include "commentscan.h"
53 #include "arguments.h"
54 #include "markdown.h"
55 #include "fileinfo.h"
56 
57 // Toggle for some debugging info
58 //#define DBG_CTX(x) fprintf x
59 #define DBG_CTX(x) do { } while(0)
60 
61 #define YY_NO_INPUT 1
62 #define YY_NO_UNISTD_H 1
63 
64 #define USE_STATE2STRING 0
65 
66 /* ----------------------------------------------------------------- */
67 
68 struct pyscannerYY_state
69 {
70  CommentScanner commentScanner;
71  OutlineParserInterface *thisParser = 0;
72  const char * inputString = 0;
73  yy_size_t inputPosition = 0;
74  Protection protection = Public;
75  std::shared_ptr<Entry> current_root;
76  std::shared_ptr<Entry> current;
77  std::shared_ptr<Entry> previous;
78  std::shared_ptr<Entry> bodyEntry;
79  int yyLineNr = 1 ;
80  QCString yyFileName;
81  MethodTypes mtype = Method;
82  bool stat = FALSE;
83  Specifier virt = Normal;
84  int docBlockContext = 0;
85  QCString docBlock;
86  bool docBlockInBody = FALSE;
87  bool docBlockJavaStyle = FALSE;
88  bool docBrief = FALSE;
89  bool docBlockSpecial = FALSE;
90  bool doubleQuote = FALSE;
91  bool specialBlock = FALSE;
92  int stringContext = 0;
93  TextStream * copyString = 0;
94  int indent = 0;
95  int curIndent = 0;
96  int commentIndent = 0;
97  bool importTuple = FALSE;
98  StringUnorderedMap packageNameCache;
99  char atomStart = 0;
100  char atomEnd = 0;
101  int atomCount = 0;
102  QCString moduleScope;
103  QCString packageName;
104  TextStream defVal;
105  int braceCount = 0;
106  bool lexInit = FALSE;
107  bool packageCommentAllowed = FALSE;
108  bool start_init = FALSE;
109  int search_count = 0;
110  QCString argType;
111  bool funcParamsEnd = FALSE;
112  std::vector<QCString> decorators;
113  QCString programStr;
114 };
115 
116 //-----------------------------------------------------------------------------
117 #if USE_STATE2STRING
118 static const char *stateToString(int state);
119 #endif
120 
121 static inline int computeIndent(const char *s);
122 
123 static void initParser(yyscan_t yyscanner);
124 static void initEntry(yyscan_t yyscanner);
125 static void newEntry(yyscan_t yyscanner);
126 static void newVariable(yyscan_t yyscanner);
127 static void newFunction(yyscan_t yyscanner);
128 static QCString findPackageScopeFromPath(yyscan_t yyscanner,const QCString &path);
129 static void addFrom(yyscan_t yyscanner,bool all);
130 static void lineCount(yyscan_t yyscanner);
131 static void incLineNr(yyscan_t yyscanner);
132 static void startCommentBlock(yyscan_t yyscanner,bool brief);
133 static void handleCommentBlock(yyscan_t yyscanner,const QCString &doc,bool brief);
134 static void endOfDef(yyscan_t yyscanner,int correction=0);
135 static inline void addToString(yyscan_t yyscanner,const char *s);
136 static void initTriDoubleQuoteBlock(yyscan_t yyscanner);
137 static void initTriSingleQuoteBlock(yyscan_t yyscanner);
138 static void initSpecialBlock(yyscan_t yyscanner);
139 static void searchFoundDef(yyscan_t yyscanner);
140 static void searchFoundClass(yyscan_t yyscanner);
141 static QCString findPackageScope(yyscan_t yyscanner,const QCString &fileName);
142 
143 static yy_size_t yyread(yyscan_t yyscanner,char *buf,yy_size_t max_size);
144 
145 //-----------------------------------------------------------------------------
146 /* ----------------------------------------------------------------- */
147 #undef YY_INPUT
148 #define YY_INPUT(buf,result,max_size) result=yyread(yyscanner,buf,max_size);
149 
150 %}
151 
152  /* start command character */
153 
154 
155 
156 BB [ \t]+
157 B [ \t]*
158 NEWLINE \n
159 BN [ \t\n]
160 
161 DIGIT [0-9]
162 
163 HEXNUMBER "0"[xX][0-9a-fA-F]+[lL]?
164 OCTNUMBER "0"[0-7]+[lL]?
165 NUMBER {DIGIT}+[lLjJ]?
166 INTNUMBER {HEXNUMBER}|{OCTNUMBER}|{NUMBER}
167 FLOATNUMBER {DIGIT}+"."{DIGIT}+([eE][+\-]?{DIGIT}+)?[jJ]?
168 BOOL ("True"|"False")
169 LETTER [A-Za-z\x80-\xFF]
170 NONEMPTY [A-Za-z0-9_\x80-\xFF]
171 EXPCHAR [#(){}\[\],:.%/\\=`*~|&<>!;+-]
172 NONEMPTYEXP [^ \t\n:]
173 PARAMNONEMPTY [^ \t\n():]
174 IDENTIFIER ({LETTER}|"_")({LETTER}|{DIGIT}|"_")*
175 SCOPE {IDENTIFIER}("."{IDENTIFIER})*
176 CALL "("[^)]*")"
177 BORDER ([^A-Za-z0-9])
178 
179 TRISINGLEQUOTE {STRINGPREFIX}?"'''"(!)?
180 TRIDOUBLEQUOTE {STRINGPREFIX}?"\"\"\""(!)?
181 ENDTRISINGLEQUOTE "'''"
182 ENDTRIDOUBLEQUOTE "\"\"\""
183 LONGSTRINGCHAR [^\\"']
184 ESCAPESEQ ("\\")(.)
185 LONGSTRINGITEM ({LONGSTRINGCHAR}|{ESCAPESEQ})
186 SMALLQUOTE ("\"\""|"\""|"'"|"''")
187 LONGSTRINGBLOCK ({LONGSTRINGITEM}|{SMALLQUOTE})
188 
189 SHORTSTRING ("'"{SHORTSTRINGITEM}*"'"|'"'{SHORTSTRINGITEM}*'"')
190 SHORTSTRINGITEM ({SHORTSTRINGCHAR}|{ESCAPESEQ})
191 SHORTSTRINGCHAR [^\\\n"]
192 STRINGLITERAL {STRINGPREFIX}?( {SHORTSTRING} | {LONGSTRING})
193 STRINGPREFIX ("r"|"u"|"ur"|"R"|"U"|"UR"|"Ur"|"uR")
194 FLOWKW ("or"|"and"|"is"|"not"|"print"|"for"|"in"|"if"|"try"|"except"|"yield"|"raise"|"break"|"continue"|"pass"|"if"|"return"|"while"|"elif"|"else"|"finally")
195 POUNDCOMMENT "#"[^#\n][^\n]*
196 SCRIPTCOMMENT "#!".*
197 
198 STARTDOCSYMS "##"
199 
200 %option noyywrap
201 
202  /* Main start state */
203 
204 %x Search
205 %x SearchMemVars
206 
207  /* Mid-comment states */
208 
209  /* %x FuncDoubleComment */
210  /* %x ClassDoubleComment */
211 %x TripleComment
212 %x SpecialComment
213 
214  /* Function states */
215 
216 %x FunctionDec
217 %x FunctionParams
218 %x FunctionBody
219 %x FunctionAnnotation
220 %x FunctionTypeAnnotation
221 %x FunctionParamDefVal
222 
223  /* Class states */
224 
225 %x ClassDec
226 %x ClassInheritance
227 %x ClassCaptureIndent
228 %x ClassBody
229 
230  /* Variable states */
231 %x VariableDec
232 %x VariableEnd
233 %x VariableAtom
234 
235  /* String states */
236 
237 %x SingleQuoteString
238 %x DoubleQuoteString
239 %x TripleString
240 %x SingleQuoteStringIgnore
241 %x DoubleQuoteStringIgnore
242 
243  /* import */
244 %x FromMod
245 %x FromModItem
246 %x Import
247 
248 %%
249 
250  /* ------------ Function recognition rules -------------- */
251 
252 <Search>{
253 
254  ^{B}"def"{BB} { // start of a function/method definition with indent
255  DBG_CTX((stderr,"Found def at %d\n",yyextra->yyLineNr));
256  yyextra->indent=computeIndent(yytext);
257  searchFoundDef(yyscanner);
258  BEGIN( FunctionDec );
259  }
260  ^{B}"async"{BB}"def"{BB} { // start of an async function/method definition with indent
261  DBG_CTX((stderr,"Found async def at %d\n",yyextra->yyLineNr));
262  yyextra->indent=computeIndent(yytext);
263  searchFoundDef(yyscanner);
264  BEGIN( FunctionDec );
265  }
266  "def"{BB} { // start of a function/method definition
267  searchFoundDef(yyscanner);
268  BEGIN( FunctionDec );
269  }
270  "async"{BB}"def"{BB} { // start of a function/method definition
271  searchFoundDef(yyscanner);
272  BEGIN( FunctionDec );
273  }
274 
275  ^{B}"class"{BB} { // start of a class definition with indent
276  DBG_CTX((stderr,"Found class at %d\n",yyextra->yyLineNr));
277  yyextra->indent=computeIndent(yytext);
278  searchFoundClass(yyscanner);
279  BEGIN( ClassDec ) ;
280  }
281  "class"{BB} { // start of a class definition
282  searchFoundClass(yyscanner);
283  BEGIN( ClassDec ) ;
284  }
285  ^{B}"from"{BB} |
286  "from"{BB} { // start of an from import
287  yyextra->packageCommentAllowed = FALSE;
288  BEGIN( FromMod );
289  }
290 
291  ^{B}"import"{BB} |
292  "import"{BB} { // start of an import statement
293  yyextra->packageCommentAllowed = FALSE;
294  BEGIN( Import );
295  }
296  ^{B}{IDENTIFIER}/{B}"="{B}"property" { // property
297  yyextra->current->section = Entry::VARIABLE_SEC;
298  yyextra->current->mtype = Property;
299  yyextra->current->name = QCString(yytext).stripWhiteSpace();
300  yyextra->current->fileName = yyextra->yyFileName;
301  yyextra->current->startLine = yyextra->yyLineNr;
302  yyextra->current->bodyLine = yyextra->yyLineNr;
303  yyextra->packageCommentAllowed = FALSE;
304  BEGIN(VariableDec);
305  }
306  ^{B}{IDENTIFIER}/{B}"="[^=] { // variable
307  if (yyextra->search_count) REJECT;
308  yyextra->indent=computeIndent(yytext);
309  yyextra->current->section = Entry::VARIABLE_SEC;
310  yyextra->current->name = QCString(yytext).stripWhiteSpace();
311  yyextra->current->fileName = yyextra->yyFileName;
312  yyextra->current->startLine = yyextra->yyLineNr;
313  yyextra->current->bodyLine = yyextra->yyLineNr;
314  yyextra->packageCommentAllowed = FALSE;
315  BEGIN(VariableDec);
316  }
317  {B}{IDENTIFIER}/({B},{B}{IDENTIFIER})*{B}")"*{B}"="[^=] { // list of variables, we cannot place the default value
318  // so we will skip it later on in a general rule
319  // Also note ")" this is to catch also (a,b). the "("
320  // is caught in the rule: [(], the ")" will be handled in [)]
321  if (yyextra->search_count > 1) REJECT;
322  yyextra->indent=computeIndent(yytext);
323  yyextra->current->section = Entry::VARIABLE_SEC;
324  yyextra->current->name = QCString(yytext).stripWhiteSpace();
325  yyextra->current->fileName = yyextra->yyFileName;
326  yyextra->current->startLine = yyextra->yyLineNr;
327  yyextra->current->bodyLine = yyextra->yyLineNr;
328  yyextra->packageCommentAllowed = FALSE;
329  newVariable(yyscanner);
330  }
331  "'" { // start of a single quoted string
332  yyextra->stringContext=YY_START;
333  yyextra->copyString=0;
334  yyextra->packageCommentAllowed = FALSE;
335  BEGIN( SingleQuoteString );
336  }
337  "\"" { // start of a double quoted string
338  yyextra->stringContext=YY_START;
339  yyextra->copyString=0;
340  yyextra->packageCommentAllowed = FALSE;
341  BEGIN( DoubleQuoteString );
342  }
343  "@staticmethod" {
344  yyextra->stat=TRUE;
345  }
346  "@"{SCOPE}{CALL}? { // decorator
347  lineCount(yyscanner);
348  }
349  {SCRIPTCOMMENT} { // Unix type script comment
350  if (yyextra->yyLineNr != 1) REJECT;
351  }
352  {POUNDCOMMENT} { // normal comment
353  yyextra->packageCommentAllowed = FALSE;
354  }
355  {IDENTIFIER} { // some other identifier
356  yyextra->packageCommentAllowed = FALSE;
357  }
358  ^{BB} {
359  yyextra->curIndent=computeIndent(yytext);
360  }
361 
362  {NEWLINE}+ { // new line
363  lineCount(yyscanner);
364  }
365 
366  {TRIDOUBLEQUOTE} { // start of a comment block
367  initTriDoubleQuoteBlock(yyscanner);
368  BEGIN(TripleComment);
369  }
370 
371  {TRISINGLEQUOTE} { // start of a comment block
372  initTriSingleQuoteBlock(yyscanner);
373  BEGIN(TripleComment);
374  }
375 
376  {STARTDOCSYMS}/[^#] { // start of a special comment
377  yyextra->curIndent=computeIndent(yytext);
378  yyextra->packageCommentAllowed = FALSE;
379  initSpecialBlock(yyscanner);
380  BEGIN(SpecialComment);
381  }
382  [(] { // we have to do something with (
383  yyextra->search_count += 1;
384  }
385  [)] { // we have to do something with )
386  yyextra->search_count -= 1;
387  }
388  [^\n] { // any other character...
389  // This is the major default
390  // that should catch everything
391  // else in Body.
392  }
393 }
394 
395 <FromMod>{
396  "." { // python3 style imports
397  }
398  {IDENTIFIER}({B}"."{B}{IDENTIFIER})* { // from package import
399  yyextra->packageName=yytext;
400  }
401  "import"{B} {
402  BEGIN(FromModItem);
403  }
404  \n {
405  incLineNr(yyscanner);
406  BEGIN(Search);
407  }
408  {B} {
409  }
410  . {
411  unput(*yytext);
412  BEGIN(Search);
413  }
414 }
415 
416 <FromModItem>{
417  "*" { // import all
418  addFrom(yyscanner,TRUE);
419  BEGIN(Search);
420  }
421  {IDENTIFIER}/{B}","{B} {
422  addFrom(yyscanner,FALSE);
423  }
424  {IDENTIFIER}/{B}")" {
425  addFrom(yyscanner,FALSE);
426  }
427  {IDENTIFIER} {
428  addFrom(yyscanner,FALSE);
429  if (!yyextra->importTuple)
430  {
431  BEGIN(Search);
432  }
433  }
434  \n {
435  incLineNr(yyscanner);
436  if (!yyextra->importTuple)
437  {
438  BEGIN(Search);
439  }
440  }
441  {B} {
442  }
443  "(" {
444  yyextra->importTuple=TRUE;
445  }
446  ")" {
447  yyextra->importTuple=FALSE;
448  BEGIN(Search);
449  }
450  "," {
451  }
452  "\\"{B}\n { // line continuation
453  incLineNr(yyscanner);
454  }
455  . {
456  unput(*yytext);
457  BEGIN(Search);
458  }
459 }
460 
461 <Import>{
462  {IDENTIFIER}({B}"."{B}{IDENTIFIER})* {
463  yyextra->current->name=removeRedundantWhiteSpace(substitute(yytext,".","::"));
464  yyextra->current->fileName = yyextra->yyFileName;
465  //printf("Adding using declaration: found:%s:%d name=%s\n",qPrint(yyextra->yyFileName),yyextra->yyLineNr,qPrint(yyextra->current->name));
466  yyextra->current->section=Entry::USINGDECL_SEC;
467  yyextra->current_root->moveToSubEntryAndRefresh(yyextra->current);
468  initEntry(yyscanner);
469  BEGIN(Search);
470  }
471  \n {
472  incLineNr(yyscanner);
473  BEGIN(Search);
474  }
475  {B} {
476  }
477  . {
478  unput(*yytext);
479  BEGIN(Search);
480  }
481 }
482 
483 <SearchMemVars>{
484  "self."{IDENTIFIER}/{B}"=" {
485  DBG_CTX((stderr,"Found instance method variable %s in %s at %d\n",&yytext[5],qPrint(yyextra->current_root->name.data(),yyextra->yyLineNr)));
486  yyextra->current->name=&yytext[5];
487  yyextra->current->section=Entry::VARIABLE_SEC;
488  yyextra->current->fileName = yyextra->yyFileName;
489  yyextra->current->startLine = yyextra->yyLineNr;
490  yyextra->current->bodyLine = yyextra->yyLineNr;
491  yyextra->current->type.resize(0);
492  if (yyextra->current->name.at(0)=='_') // mark as private
493  {
494  yyextra->current->protection=Private;
495  }
496  newEntry(yyscanner);
497  }
498  "cls."{IDENTIFIER}/{B}"=" {
499  DBG_CTX((stderr,"Found class method variable %s in %s at %d\n",&yytext[4],qPrint(yyextra->current_root->name),yyextra->yyLineNr));
500  yyextra->current->name=&yytext[4];
501  yyextra->current->section=Entry::VARIABLE_SEC;
502  yyextra->current->fileName = yyextra->yyFileName;
503  yyextra->current->startLine = yyextra->yyLineNr;
504  yyextra->current->bodyLine = yyextra->yyLineNr;
505  yyextra->current->type.resize(0);
506  if (yyextra->current->name.at(0)=='_') // mark as private
507  {
508  yyextra->current->protection=Private;
509  }
510  newEntry(yyscanner);
511  }
512  {TRIDOUBLEQUOTE} { // start of a comment block
513  initTriDoubleQuoteBlock(yyscanner);
514  BEGIN(TripleComment);
515  }
516 
517  {TRISINGLEQUOTE} { // start of a comment block
518  initTriSingleQuoteBlock(yyscanner);
519  BEGIN(TripleComment);
520  }
521 
522  {STARTDOCSYMS}/[^#] { // start of a special comment
523  initSpecialBlock(yyscanner);
524  BEGIN(SpecialComment);
525  }
526  {POUNDCOMMENT} { // #
527  }
528  "'" { // start of a single quoted string
529  yyextra->stringContext=YY_START;
530  yyextra->copyString=0;
531  BEGIN( SingleQuoteString );
532  }
533  "\"" { // start of a double quoted string
534  yyextra->stringContext=YY_START;
535  yyextra->copyString=0;
536  BEGIN( DoubleQuoteString );
537  }
538  \n { incLineNr(yyscanner); }
539  {IDENTIFIER} // identifiers
540  [^'"\.#a-z_A-Z\n]+ // other uninteresting stuff
541  . // anything else
542 }
543 
544 <FunctionBody>{
545  \n{B}/{IDENTIFIER}{BB} {
546  DBG_CTX((stderr,"indent %d<=%d\n",computeIndent(&yytext[1]),yyextra->indent));
547  if (computeIndent(&yytext[1])<=yyextra->indent)
548  {
549  int i;
550  for (i=(int)yyleng-1;i>=0;i--)
551  {
552  unput(yytext[i]);
553  }
554  endOfDef(yyscanner);
555  //YY_CURRENT_BUFFER->yy_at_bol=TRUE;
556  BEGIN(Search);
557  }
558  else
559  {
560  incLineNr(yyscanner);
561  yyextra->current->program << yytext;
562  }
563  }
564  \n{B}/"##" {
565  if (computeIndent(&yytext[1])<=yyextra->indent)
566  {
567  int i;
568  for (i=(int)yyleng-1;i>=0;i--)
569  {
570  unput(yytext[i]);
571  }
572  endOfDef(yyscanner);
573  //YY_CURRENT_BUFFER->yy_at_bol=TRUE;
574  BEGIN(Search);
575  }
576  else
577  {
578  incLineNr(yyscanner);
579  yyextra->current->program << yytext;
580  }
581  }
582  <<EOF>> {
583  endOfDef(yyscanner);
584  yyterminate();
585  }
586  ^{BB}/\n { // skip empty line
587  yyextra->current->program << yytext;
588  }
589  ^{BB} { // something at indent >0
590  yyextra->current->program << yytext;
591  yyextra->curIndent = computeIndent(yytext);
592  if (yyextra->curIndent<=yyextra->indent)
593  // jumped out of the function
594  {
595  endOfDef(yyscanner,1);
596  BEGIN(Search);
597  }
598  }
599  "'" { // start of a single quoted string
600  yyextra->current->program << yytext;
601  yyextra->stringContext=YY_START;
602  yyextra->specialBlock = FALSE;
603  yyextra->copyString=&yyextra->current->program;
604  BEGIN( SingleQuoteString );
605  }
606  "\"" { // start of a double quoted string
607  yyextra->current->program << yytext;
608  yyextra->stringContext=YY_START;
609  yyextra->specialBlock = FALSE;
610  yyextra->copyString=&yyextra->current->program;
611  BEGIN( DoubleQuoteString );
612  }
613  [^ \t\n#'".]+ { // non-special stuff
614  yyextra->current->program << yytext;
615  yyextra->specialBlock = FALSE;
616  }
617  ^{POUNDCOMMENT} { // normal comment
618  yyextra->current->program << yytext;
619  }
620  "#".* { // comment half way
621  yyextra->current->program << yytext;
622  }
623  {NEWLINE} {
624  incLineNr(yyscanner);
625  yyextra->current->program << yytext;
626  }
627  . { // any character
628  yyextra->current->program << *yytext;
629  yyextra->specialBlock = FALSE;
630  }
631 
632  {TRIDOUBLEQUOTE} { // start of a comment block
633  yyextra->current->program << yytext;
634  initTriDoubleQuoteBlock(yyscanner);
635  BEGIN(TripleComment);
636  }
637 
638  {TRISINGLEQUOTE} { // start of a comment block
639  yyextra->current->program << yytext;
640  initTriSingleQuoteBlock(yyscanner);
641  BEGIN(TripleComment);
642  }
643 
644  {STARTDOCSYMS}/[^#] { // start of a special comment
645  initSpecialBlock(yyscanner);
646  BEGIN(SpecialComment);
647  }
648 
649 }
650 
651 <FunctionDec>{
652  {IDENTIFIER} {
653  //found function name
654  if (yyextra->current->type.isEmpty())
655  {
656  yyextra->current->type = "def";
657  }
658  yyextra->current->name = yytext;
659  yyextra->current->name = yyextra->current->name.stripWhiteSpace();
660  newFunction(yyscanner);
661  }
662  {B}":"{B} { // function without arguments
663  yyextra->specialBlock = TRUE; // expecting a docstring
664  yyextra->bodyEntry = yyextra->current;
665  yyextra->current->bodyLine = yyextra->yyLineNr;
666  BEGIN(FunctionBody);
667  }
668 
669  "->" {
670  yyextra->defVal.str(std::string());
671  yyextra->braceCount = 0;
672  BEGIN(FunctionTypeAnnotation);
673  }
674  {B}"(" {
675  yyextra->funcParamsEnd = FALSE;
676  yyextra->current->bodyLine = yyextra->yyLineNr;
677  BEGIN(FunctionParams);
678  }
679  ")" { // end of parameter list
680  if (yyextra->current->argList.empty())
681  {
682  yyextra->current->argList.setNoParameters(TRUE);
683  }
684  yyextra->current->args = argListToString(yyextra->current->argList);
685  yyextra->funcParamsEnd = TRUE;
686  }
687 }
688 
689 <FunctionParams>{
690  ({BB}|",") {
691  }
692 
693  [\*]+ {
694  yyextra->argType = yytext;
695  }
696  {IDENTIFIER} { // Name of parameter
697  lineCount(yyscanner);
698  Argument a;
699  a.name = QCString(yytext).stripWhiteSpace();
700  a.type = yyextra->argType;
701  yyextra->current->argList.push_back(a);
702  yyextra->argType = "";
703  }
704  "=" { // default value
705  // TODO: this rule is too simple, need to be able to
706  // match things like =")" as well!
707  yyextra->defVal.str(std::string());
708  yyextra->braceCount = 0;
709  BEGIN(FunctionParamDefVal);
710  }
711  ")" {
712  unput(*yytext);
713  BEGIN(FunctionDec);
714  }
715  ":"{B} {
716  yyextra->defVal.str(std::string());
717  yyextra->braceCount = 0;
718  BEGIN(FunctionAnnotation);
719  }
720  {POUNDCOMMENT} { // a comment
721  }
722  {PARAMNONEMPTY} { // Default rule inside arguments.
723  }
724 
725 }
726 
727 <FunctionTypeAnnotation>{
728  "{" |
729  "[" |
730  "(" {
731  ++yyextra->braceCount;
732  yyextra->defVal << *yytext;
733  }
734  "}" |
735  "]" |
736  ")" {
737  --yyextra->braceCount;
738  yyextra->defVal << *yytext;
739  }
740  ":" {
741  if (yyextra->braceCount == 0)
742  {
743  yyextra->current->type = yyextra->defVal.str();
744  unput(*yytext);
745  BEGIN(FunctionDec);
746  }
747  else
748  yyextra->defVal << *yytext;
749  }
750  "'" {
751  yyextra->defVal << *yytext;
752  yyextra->copyString=&yyextra->defVal;
753  yyextra->stringContext=FunctionTypeAnnotation;
754  BEGIN(SingleQuoteString);
755  }
756  "\"" {
757  yyextra->defVal << *yytext;
758  yyextra->copyString=&yyextra->defVal;
759  yyextra->stringContext=FunctionTypeAnnotation;
760  BEGIN(DoubleQuoteString);
761  }
762  \n {
763  yyextra->defVal << *yytext;
764  incLineNr(yyscanner);
765  }
766  . {
767  yyextra->defVal << *yytext;
768  }
769 }
770 
771 <FunctionAnnotation>{
772  "{" |
773  "[" |
774  "(" {
775  ++yyextra->braceCount;
776  yyextra->defVal << *yytext;
777  }
778  "}" |
779  "]" {
780  --yyextra->braceCount;
781  yyextra->defVal << *yytext;
782  }
783  ")" |
784  "=" |
785  "," {
786  if (yyextra->braceCount == 0)
787  {
788  if (!yyextra->current->argList.empty())
789  yyextra->current->argList.back().type += yyextra->defVal.str();
790  if (*yytext != ',')
791  unput(*yytext);
792  BEGIN(FunctionParams);
793  }
794  else
795  {
796  if (*yytext == ')')
797  --yyextra->braceCount;
798  yyextra->defVal << *yytext;
799  }
800  }
801  "'" {
802  yyextra->defVal << *yytext;
803  yyextra->copyString=&yyextra->defVal;
804  yyextra->stringContext=FunctionAnnotation;
805  BEGIN(SingleQuoteString);
806  }
807  "\"" {
808  yyextra->defVal << *yytext;
809  yyextra->copyString=&yyextra->defVal;
810  yyextra->stringContext=FunctionAnnotation;
811  BEGIN(DoubleQuoteString);
812  }
813  \n {
814  yyextra->defVal << *yytext;
815  incLineNr(yyscanner);
816  }
817  . {
818  yyextra->defVal << *yytext;
819  }
820 }
821 
822 <FunctionParamDefVal>{
823  "{" |
824  "[" |
825  "(" { // internal opening brace, assumption is that we have correct code so braces do match
826  ++yyextra->braceCount;
827  yyextra->defVal << *yytext;
828  }
829  "}" |
830  "]" {
831  --yyextra->braceCount;
832  yyextra->defVal << *yytext;
833  }
834  ")" |
835  "," {
836  if (yyextra->braceCount == 0)
837  {
838  if (!yyextra->current->argList.empty())
839  yyextra->current->argList.back().defval=QCString(yyextra->defVal.str()).stripWhiteSpace();
840  if (*yytext == ')')
841  unput(*yytext);
842  BEGIN(FunctionParams);
843  }
844  else
845  {
846  if (*yytext == ')')
847  --yyextra->braceCount;
848  yyextra->defVal << *yytext;
849  }
850  }
851 
852  "'" {
853  yyextra->defVal << *yytext;
854  yyextra->copyString=&yyextra->defVal;
855  yyextra->stringContext=FunctionParamDefVal;
856  BEGIN( SingleQuoteString );
857  }
858  "\"" {
859  yyextra->defVal << *yytext;
860  yyextra->copyString=&yyextra->defVal;
861  yyextra->stringContext=FunctionParamDefVal;
862  BEGIN( DoubleQuoteString );
863  }
864  \n {
865  yyextra->defVal << *yytext;
866  incLineNr(yyscanner);
867  }
868  . {
869  yyextra->defVal << *yytext;
870  }
871 }
872 
873 
874 <ClassBody>{
875  \n/{IDENTIFIER}{BB} { // new def at indent 0
876  incLineNr(yyscanner);
877  endOfDef(yyscanner);
878  //yyextra->hideClassDocs = FALSE;
879  //YY_CURRENT_BUFFER->yy_at_bol=TRUE;
880  BEGIN(Search);
881  }
882  \n/"##"[^#] { // start of a special comment at indent 0
883  incLineNr(yyscanner);
884  endOfDef(yyscanner);
885  //yyextra->hideClassDocs = FALSE;
886  //YY_CURRENT_BUFFER->yy_at_bol=TRUE;
887  BEGIN(Search);
888  }
889  ^{BB}/\n { // skip empty line
890  yyextra->current->program << yytext;
891  }
892  <<EOF>> {
893  endOfDef(yyscanner);
894  yyterminate();
895  }
896  ^{BB} { // something at indent >0
897  yyextra->curIndent=computeIndent(yytext);
898  DBG_CTX((stderr,"yyextra->curIndent=%d yyextra->indent=%d\n",yyextra->curIndent,yyextra->indent));
899  if (yyextra->curIndent<=yyextra->indent)
900  // jumped out of the class/method
901  {
902  endOfDef(yyscanner,1);
903  yyextra->indent=yyextra->curIndent;
904  // make sure the next rule matches ^...
905  //YY_CURRENT_BUFFER->yy_at_bol=TRUE;
906  //yyextra->hideClassDocs = FALSE;
907  BEGIN(Search);
908  }
909  else
910  {
911  yyextra->current->program << yytext;
912  }
913  }
914  "'" { // start of a single quoted string
915  yyextra->current->program << *yytext;
916  yyextra->stringContext=YY_START;
917  yyextra->specialBlock = FALSE;
918  yyextra->copyString=&yyextra->current->program;
919  BEGIN( SingleQuoteString );
920  }
921  "\"" { // start of a double quoted string
922  yyextra->current->program << *yytext;
923  yyextra->stringContext=YY_START;
924  yyextra->specialBlock = FALSE;
925  yyextra->copyString=&yyextra->current->program;
926  BEGIN( DoubleQuoteString );
927  }
928  [^ \t\n#'"]+ { // non-special stuff
929  yyextra->current->program << yytext;
930  yyextra->specialBlock = FALSE;
931  //yyextra->hideClassDocs = FALSE;
932  }
933  {NEWLINE} {
934  yyextra->current->program << *yytext;
935  incLineNr(yyscanner);
936  }
937  {POUNDCOMMENT} { // normal comment
938  yyextra->current->program << yytext;
939  }
940  . { // any character
941  yyextra->specialBlock = FALSE;
942  yyextra->current->program << *yytext;
943  }
944  {TRIDOUBLEQUOTE} { // start of a comment block
945  //if (!yyextra->hideClassDocs)
946  yyextra->current->program << yytext;
947  initTriDoubleQuoteBlock(yyscanner);
948  BEGIN(TripleComment);
949  }
950 
951  {TRISINGLEQUOTE} { // start of a comment block
952  //if (!yyextra->hideClassDocs)
953  yyextra->current->program << yytext;
954  initTriSingleQuoteBlock(yyscanner);
955  BEGIN(TripleComment);
956  }
957 }
958 
959 <ClassDec>{IDENTIFIER} {
960  if (yyextra->current->type.isEmpty())
961  {
962  yyextra->current->type = "class";
963  }
964 
965  yyextra->current->section = Entry::CLASS_SEC;
966  yyextra->current->name = yytext;
967 
968  // prepend scope in case of nested classes
969  if (yyextra->current_root->section&Entry::SCOPE_MASK)
970  {
971  //printf("*** Prepending scope %s to class %s\n",qPrint(yyextra->current_root->name),qPrint(yyextra->current->name));
972  yyextra->current->name.prepend(yyextra->current_root->name+"::");
973  }
974 
975  yyextra->current->name = yyextra->current->name.stripWhiteSpace();
976  yyextra->current->fileName = yyextra->yyFileName;
977  yyextra->docBlockContext = YY_START;
978  yyextra->docBlockInBody = FALSE;
979  yyextra->docBlockJavaStyle = FALSE;
980  yyextra->docBlock.resize(0);
981 
982  BEGIN(ClassInheritance);
983  }
984 
985 <ClassInheritance>{
986  ({BB}|[\(,\)]) { // syntactic sugar for the list
987  }
988 
989  ":" { // begin of the class definition
990  yyextra->specialBlock = TRUE; // expecting a docstring
991  yyextra->current->bodyLine = yyextra->yyLineNr;
992  yyextra->current->program.str(std::string());
993  BEGIN(ClassCaptureIndent);
994  }
995 
996  {SCOPE} {
997  yyextra->current->extends.push_back(
998  BaseInfo(substitute(yytext,".","::"),Public,Normal)
999  );
1000  //Has base class-do stuff
1001  }
1002  "'" { // start of a single quoted string
1003  yyextra->stringContext=YY_START;
1004  BEGIN( SingleQuoteStringIgnore );
1005  }
1006  "\"" { // start of a double quoted string
1007  yyextra->stringContext=YY_START;
1008  BEGIN( DoubleQuoteStringIgnore );
1009  }
1010 }
1011 
1012 <SingleQuoteStringIgnore>{
1013  "'" { // end of a single quoted string
1014  BEGIN(yyextra->stringContext);
1015  }
1016  . { }
1017 }
1018 <DoubleQuoteStringIgnore>{
1019  "\"" { // end of a double quoted string
1020  BEGIN(yyextra->stringContext);
1021  }
1022  . { }
1023 }
1024 
1025 <ClassCaptureIndent>{
1026  "\n"|({BB}"\n") {
1027  // Blankline - ignore, keep looking for indentation.
1028  lineCount(yyscanner);
1029  yyextra->current->program << yytext;
1030  }
1031 
1032  {TRIDOUBLEQUOTE} { // start of a comment block
1033  initTriDoubleQuoteBlock(yyscanner);
1034  yyextra->current->program << yytext;
1035  BEGIN(TripleComment);
1036  }
1037  {TRISINGLEQUOTE} { // start of a comment block
1038  initTriSingleQuoteBlock(yyscanner);
1039  yyextra->current->program << yytext;
1040  BEGIN(TripleComment);
1041  }
1042  {STARTDOCSYMS}[#]* { // start of a special comment
1043  initSpecialBlock(yyscanner);
1044  BEGIN(SpecialComment);
1045  }
1046  {POUNDCOMMENT} { // ignore comment with just one #
1047  }
1048  ^{BB} {
1049  yyextra->current->program << yytext;
1050  //yyextra->current->startLine = yyextra->yyLineNr;
1051  yyextra->curIndent=computeIndent(yytext);
1052  yyextra->bodyEntry = yyextra->current;
1053  DBG_CTX((stderr,"setting indent %d\n",yyextra->curIndent));
1054  //printf("yyextra->current->program=[%s]\n",qPrint(yyextra->current->program));
1055  //yyextra->hideClassDocs = TRUE;
1056  BEGIN(ClassBody);
1057  }
1058 
1059  ""/({NONEMPTY}|{EXPCHAR}) {
1060  // Just pushback an empty class, and
1061  // resume parsing the body.
1062  newEntry(yyscanner);
1063  yyextra->current->program << yytext;
1064 
1065  // printf("Failed to find indent - skipping!");
1066  BEGIN( Search );
1067  }
1068 }
1069 
1070 
1071 <VariableDec>{
1072  "=" { // the assignment operator
1073  //printf("====== VariableDec at line %d\n",yyextra->yyLineNr);
1074  yyextra->start_init = TRUE;
1075  yyextra->current->initializer.str(yytext);
1076  yyextra->current->initializer << " ";
1077  }
1078  {B} { // spaces
1079  yyextra->current->initializer << yytext;
1080  }
1081  {INTNUMBER} { // integer value
1082  if (yyextra->current-> type.isEmpty()) yyextra->current->type = "int";
1083  yyextra->current->initializer << yytext;
1084  }
1085  {FLOATNUMBER} { // floating point value
1086  if (yyextra->current->type.isEmpty()) yyextra->current->type = "float";
1087  yyextra->current->initializer << yytext;
1088  }
1089  {BOOL} { // boolean value
1090  if (yyextra->current->type.isEmpty()) yyextra->current->type = "bool";
1091  yyextra->current->initializer << yytext;
1092  }
1093  {STRINGPREFIX}?"'" { // string
1094  if (yyextra->current->type.isEmpty()) yyextra->current->type = "string";
1095  yyextra->current->initializer << yytext;
1096  yyextra->copyString=&yyextra->current->initializer;
1097  yyextra->stringContext=VariableDec;
1098  BEGIN( SingleQuoteString );
1099  }
1100  {STRINGPREFIX}?"\"" { // string
1101  if (yyextra->current->type.isEmpty()) yyextra->current->type = "string";
1102  yyextra->current->initializer << yytext;
1103  yyextra->copyString=&yyextra->current->initializer;
1104  yyextra->stringContext=VariableDec;
1105  BEGIN( DoubleQuoteString );
1106  }
1107  {TRIDOUBLEQUOTE} { // start of a comment block
1108  if (yyextra->current->type.isEmpty()) yyextra->current->type = "string";
1109  yyextra->current->initializer << yytext;
1110  yyextra->doubleQuote=TRUE;
1111  yyextra->copyString=&yyextra->current->initializer;
1112  yyextra->stringContext=VariableDec;
1113  BEGIN(TripleString);
1114  }
1115 
1116  {TRISINGLEQUOTE} { // start of a comment block
1117  if (yyextra->current->type.isEmpty()) yyextra->current->type = "string";
1118  yyextra->current->initializer << yytext;
1119  yyextra->doubleQuote=FALSE;
1120  yyextra->copyString=&yyextra->current->initializer;
1121  yyextra->stringContext=VariableDec;
1122  BEGIN(TripleString);
1123  }
1124  "(" { // tuple, only when direct after =
1125  if (yyextra->current->mtype!=Property && yyextra->start_init)
1126  {
1127  yyextra->current->type = "tuple";
1128  }
1129  yyextra->current->initializer << *yytext;
1130  yyextra->atomStart='(';
1131  yyextra->atomEnd=')';
1132  yyextra->atomCount=1;
1133  BEGIN( VariableAtom );
1134  }
1135  "[" { // list
1136  if (yyextra->start_init) yyextra->current->type = "list";
1137  yyextra->current->initializer << *yytext;
1138  yyextra->atomStart='[';
1139  yyextra->atomEnd=']';
1140  yyextra->atomCount=1;
1141  BEGIN( VariableAtom );
1142  }
1143  "{" { // dictionary
1144  if (yyextra->start_init) yyextra->current->type = "dictionary";
1145  yyextra->current->initializer << *yytext;
1146  yyextra->atomStart='{';
1147  yyextra->atomEnd='}';
1148  yyextra->atomCount=1;
1149  BEGIN( VariableAtom );
1150  }
1151  "#".* { // comment
1152  BEGIN( VariableEnd );
1153  }
1154  {IDENTIFIER} {
1155  // do something based on the type of the IDENTIFIER
1156  if (yyextra->current->type.isEmpty())
1157  {
1158  for (const auto &child : yyextra->current_root->children())
1159  {
1160  if (child->name == QCString(yytext))
1161  {
1162  yyextra->current->type = child->type;
1163  break;
1164  }
1165  }
1166  }
1167  yyextra->start_init = FALSE;
1168  yyextra->current->initializer << yytext;
1169  }
1170  . {
1171  yyextra->start_init = FALSE;
1172  yyextra->current->initializer << *yytext;
1173  }
1174  \n {
1175  unput('\n');
1176  BEGIN( VariableEnd );
1177  }
1178 }
1179 
1180 <VariableAtom>{
1181  [\(\[\{] {
1182  yyextra->current->initializer << *yytext;
1183  if (yyextra->atomStart==*yytext)
1184  {
1185  yyextra->atomCount++;
1186  }
1187  }
1188  [\)\]\}] {
1189  yyextra->current->initializer << *yytext;
1190  if (yyextra->atomEnd==*yytext)
1191  {
1192  yyextra->atomCount--;
1193  }
1194  if (yyextra->atomCount==0)
1195  {
1196  yyextra->start_init = FALSE;
1197  BEGIN(VariableDec);
1198  }
1199  }
1200  {TRIDOUBLEQUOTE} { // start of a comment block
1201  yyextra->specialBlock = FALSE;
1202  yyextra->current->program << yytext;
1203  initTriDoubleQuoteBlock(yyscanner);
1204  BEGIN(TripleComment);
1205  }
1206 
1207  {TRISINGLEQUOTE} { // start of a comment block
1208  yyextra->specialBlock = FALSE;
1209  yyextra->current->program << yytext;
1210  initTriSingleQuoteBlock(yyscanner);
1211  BEGIN(TripleComment);
1212  }
1213  "'" {
1214  yyextra->stringContext=YY_START;
1215  yyextra->current->initializer << "'";
1216  yyextra->copyString=&yyextra->current->initializer;
1217  BEGIN( SingleQuoteString );
1218  }
1219  "\"" {
1220  yyextra->stringContext=YY_START;
1221  yyextra->current->initializer << "\"";
1222  yyextra->copyString=&yyextra->current->initializer;
1223  BEGIN( DoubleQuoteString );
1224  }
1225  {IDENTIFIER} {
1226  yyextra->current->initializer << yytext;
1227  }
1228  . {
1229  yyextra->current->initializer << *yytext;
1230  }
1231  \n {
1232  yyextra->current->initializer << *yytext;
1233  incLineNr(yyscanner);
1234  }
1235 
1236 }
1237 
1238 <VariableEnd>{
1239  \n {
1240  incLineNr(yyscanner);
1241  newVariable(yyscanner);
1242  BEGIN(Search);
1243  }
1244  . {
1245  unput(*yytext);
1246  newVariable(yyscanner);
1247  BEGIN(Search);
1248  }
1249  <<EOF>> { yyterminate();
1250  }
1251 }
1252 
1253 <TripleComment>{
1254  {ENDTRIDOUBLEQUOTE} |
1255  {ENDTRISINGLEQUOTE} {
1256  // printf("Expected module block %d special=%d\n",yyextra->expectModuleDocs,yyextra->specialBlock);
1257  if (yyextra->doubleQuote==(yytext[0]=='"'))
1258  {
1259  if (yyextra->specialBlock) // expecting a docstring
1260  {
1261  QCString actualDoc=yyextra->docBlock;
1262  if (!yyextra->docBlockSpecial) // legacy unformatted docstring
1263  {
1264  if (!actualDoc.isEmpty())
1265  {
1266  stripIndentation(actualDoc,yyextra->commentIndent);
1267  actualDoc.prepend("\\verbatim\n");
1268  actualDoc.append("\\endverbatim ");
1269  }
1270  }
1271  //printf("-------> yyextra->current=%p yyextra->bodyEntry=%p\n",yyextra->current,yyextra->bodyEntry);
1272  handleCommentBlock(yyscanner, actualDoc, FALSE);
1273  }
1274  else if (yyextra->packageCommentAllowed) // expecting module docs
1275  {
1276  QCString actualDoc=yyextra->docBlock;
1277  if (!yyextra->docBlockSpecial) // legacy unformatted docstring
1278  {
1279  if (!actualDoc.isEmpty())
1280  {
1281  stripIndentation(actualDoc,yyextra->commentIndent);
1282  actualDoc.prepend("\\verbatim\n");
1283  actualDoc.append("\\endverbatim ");
1284  }
1285  }
1286  if (yyextra->moduleScope.startsWith("__") && yyextra->moduleScope.endsWith("__"))
1287  {
1288  actualDoc.prepend("\\namespace \\"+yyextra->moduleScope+" ");
1289  }
1290  else
1291  {
1292  actualDoc.prepend("\\namespace "+yyextra->moduleScope+" ");
1293  }
1294  handleCommentBlock(yyscanner, actualDoc, FALSE);
1295  }
1296  if ((yyextra->docBlockContext==ClassBody /*&& !yyextra->hideClassDocs*/) ||
1297  yyextra->docBlockContext==FunctionBody)
1298  {
1299  yyextra->current->program << yyextra->docBlock;
1300  yyextra->current->program << yytext;
1301  }
1302  //if (yyextra->hideClassDocs)
1303  //{
1304  // yyextra->current->startLine = yyextra->yyLineNr;
1305  //}
1306  //yyextra->hideClassDocs=FALSE;
1307  BEGIN(yyextra->docBlockContext);
1308  }
1309  else
1310  {
1311  yyextra->docBlock += yytext;
1312  }
1313  yyextra->packageCommentAllowed = FALSE;
1314  }
1315 
1316 
1317  ^{BB} { // leading whitespace
1318  yyextra->docBlock += yytext;
1319  }
1320  [^"'\n \t\\]+ {
1321  yyextra->docBlock += yytext;
1322  }
1323  \n {
1324  incLineNr(yyscanner);
1325  yyextra->docBlock += yytext;
1326  }
1327  \\. { // escaped char
1328  yyextra->docBlock += yytext;
1329  }
1330  . {
1331  yyextra->docBlock += yytext;
1332  }
1333 }
1334 
1335 <SpecialComment>{
1336  ^{B}"#"("#")* { // skip leading hashes
1337  }
1338  \n/{B}"#" { // continuation of the comment on the next line
1339  yyextra->docBlock+='\n';
1340  yyextra->docBrief = FALSE;
1341  incLineNr(yyscanner);
1342  }
1343  [^#\n]+ { // any other stuff
1344  yyextra->docBlock+=yytext;
1345  }
1346  \n { // new line that ends the comment
1347  handleCommentBlock(yyscanner, yyextra->docBlock, yyextra->docBrief);
1348  incLineNr(yyscanner);
1349  BEGIN(yyextra->docBlockContext);
1350  }
1351  . { // anything we missed
1352  yyextra->docBlock+=*yytext;
1353  }
1354 }
1355 
1356 <SingleQuoteString>{
1357  \\{B}\n { // line continuation
1358  addToString(yyscanner,yytext);
1359  incLineNr(yyscanner);
1360  }
1361  \\. { // escaped char
1362  addToString(yyscanner,yytext);
1363  }
1364  "\"\"\"" { // triple double quotes
1365  addToString(yyscanner,yytext);
1366  }
1367  "'" { // end of the string
1368  addToString(yyscanner,yytext);
1369  BEGIN(yyextra->stringContext);
1370  }
1371  [^"'\n\\]+ { // normal chars
1372  addToString(yyscanner,yytext);
1373  }
1374  . { // normal char
1375  addToString(yyscanner,yytext);
1376  }
1377 }
1378 
1379 <DoubleQuoteString>{
1380  \\{B}\n { // line continuation
1381  addToString(yyscanner,yytext);
1382  incLineNr(yyscanner);
1383  }
1384  \\. { // escaped char
1385  addToString(yyscanner,yytext);
1386  }
1387  "'''" { // triple single quotes
1388  addToString(yyscanner,yytext);
1389  }
1390  "\"" { // end of the string
1391  addToString(yyscanner,yytext);
1392  BEGIN(yyextra->stringContext);
1393  }
1394  [^"'\n\\]+ { // normal chars
1395  addToString(yyscanner,yytext);
1396  }
1397  . { // normal char
1398  addToString(yyscanner,yytext);
1399  }
1400 }
1401 
1402 <TripleString>{
1403  {ENDTRIDOUBLEQUOTE} |
1404  {ENDTRISINGLEQUOTE} {
1405  *yyextra->copyString << yytext;
1406  if (yyextra->doubleQuote==(yytext[0]=='"'))
1407  {
1408  BEGIN(yyextra->stringContext);
1409  }
1410  }
1411 
1412 
1413  ({LONGSTRINGBLOCK}) {
1414  lineCount(yyscanner);
1415  *yyextra->copyString << yytext;
1416  }
1417  \n {
1418  incLineNr(yyscanner);
1419  *yyextra->copyString << yytext;
1420  }
1421  . {
1422  *yyextra->copyString << *yytext;
1423  }
1424 }
1425 
1426  /* ------------ End rules -------------- */
1427 
1428  /*
1429 <*>({NONEMPTY}|{EXPCHAR}|{BB}) { // This should go one character at a time.
1430  // printf("[pyscanner] '%s' [ state %d ] [line %d] no match\n",
1431  // yytext, YY_START, yyextra->yyLineNr);
1432 
1433  }
1434  */
1435 
1436 <*>{NEWLINE} {
1437  //printf("[pyscanner] %d NEWLINE [line %d] no match\n",
1438  // YY_START, yyextra->yyLineNr);
1439 
1440  lineCount(yyscanner);
1441  }
1442 
1443 <*>"'" {
1444  //fprintf(stderr,"Quote: %d\n",YY_START);
1445  }
1446 
1447 <*>. {
1448  //printf("[pyscanner] '%s' [ state %d ] [line %d] no match\n",
1449  // yytext, YY_START, yyextra->yyLineNr);
1450 
1451  }
1452 
1453 
1454 %%
1455 
1456 //----------------------------------------------------------------------------
1457 
1458 static yy_size_t yyread(yyscan_t yyscanner,char *buf,yy_size_t max_size)
1459 {
1460  struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1461  yy_size_t c=0;
1462  const char *p = yyextra->inputString + yyextra->inputPosition;
1463  while ( c < max_size && *p ) { *buf++ = *p++; c++; }
1464  yyextra->inputPosition+=c;
1465  return c;
1466 }
1467 
1468 static void initParser(yyscan_t yyscanner)
1469 {
1470  struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1471  yyextra->protection = Public;
1472  yyextra->mtype = Method;
1473  yyextra->stat = FALSE;
1474  yyextra->virt = Normal;
1475  yyextra->previous = 0;
1476  yyextra->packageCommentAllowed = TRUE;
1477 }
1478 
1479 static void initEntry(yyscan_t yyscanner)
1480 {
1481  struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1482  //yyextra->current->python = TRUE;
1483  yyextra->current->protection = yyextra->protection ;
1484  yyextra->current->mtype = yyextra->mtype;
1485  yyextra->current->virt = yyextra->virt;
1486  yyextra->current->stat = yyextra->stat;
1487  yyextra->current->lang = SrcLangExt_Python;
1488  yyextra->commentScanner.initGroupInfo(yyextra->current.get());
1489  yyextra->stat = FALSE;
1490 }
1491 
1492 static void newEntry(yyscan_t yyscanner)
1493 {
1494  struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1495  yyextra->previous = yyextra->current;
1496  yyextra->current_root->moveToSubEntryAndRefresh(yyextra->current);
1497  initEntry(yyscanner);
1498 }
1499 
1500 static void newVariable(yyscan_t yyscanner)
1501 {
1502  struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1503  if (!yyextra->current->name.isEmpty() && yyextra->current->name.at(0)=='_') // mark as private
1504  {
1505  yyextra->current->protection=Private;
1506  }
1507  if (yyextra->current_root->section&Entry::COMPOUND_MASK) // mark as class variable
1508  {
1509  yyextra->current->stat = TRUE;
1510  }
1511  newEntry(yyscanner);
1512 }
1513 
1514 static void newFunction(yyscan_t yyscanner)
1515 {
1516  struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1517  if (yyextra->current->name.left(2)=="__" && yyextra->current->name.right(2)=="__")
1518  {
1519  // special method name, see
1520  // http://docs.python.org/ref/specialnames.html
1521  yyextra->current->protection=Public;
1522  }
1523  else if (yyextra->current->name.at(0)=='_')
1524  {
1525  yyextra->current->protection=Private;
1526  }
1527 }
1528 
1529 static inline int computeIndent(const char *s)
1530 {
1531  int col=0;
1532  int tabSize=Config_getInt(TAB_SIZE);
1533  const char *p=s;
1534  char c;
1535  while ((c=*p++))
1536  {
1537  if (c==' ') col++;
1538  else if (c=='\t') col+=tabSize-(col%tabSize);
1539  else break;
1540  }
1541  return col;
1542 }
1543 
1544 static QCString findPackageScopeFromPath(yyscan_t yyscanner,const QCString &path)
1545 {
1546  struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1547  auto it = yyextra->packageNameCache.find(path.str());
1548  if (it!=yyextra->packageNameCache.end())
1549  {
1550  return QCString(it->second);
1551  }
1552  FileInfo pf(path.str()+"/__init__.py"); // found package initialization file
1553  if (pf.exists())
1554  {
1555  int i=path.findRev('/');
1556  if (i!=-1)
1557  {
1558  QCString scope = findPackageScopeFromPath(yyscanner,path.left(i));
1559  if (!scope.isEmpty())
1560  {
1561  scope+="::";
1562  }
1563  scope+=path.mid(i+1);
1564  yyextra->packageNameCache.insert(std::make_pair(path.str(),scope.str()));
1565  return scope;
1566  }
1567  }
1568  return "";
1569 }
1570 
1571 static QCString findPackageScope(yyscan_t yyscanner,const QCString &fileName)
1572 {
1573  if (fileName.isEmpty()) return fileName;
1574  FileInfo fi(fileName.str());
1575  return findPackageScopeFromPath(yyscanner,fi.dirPath(true).c_str());
1576 }
1577 
1578 static void addFrom(yyscan_t yyscanner,bool all)
1579 {
1580  struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1581  QCString item=all ? yyextra->packageName : yyextra->packageName+"."+yytext;
1582  yyextra->current->name=removeRedundantWhiteSpace(substitute(item,".","::"));
1583  yyextra->current->fileName = yyextra->yyFileName;
1584  //printf("Adding using declaration: found:%s:%d name=%s\n",qPrint(yyextra->yyFileName),yyextra->yyLineNr,qPrint(yyextra->current->name));
1585  yyextra->current->section=all ? Entry::USINGDIR_SEC : Entry::USINGDECL_SEC;
1586  yyextra->current_root->moveToSubEntryAndRefresh(yyextra->current);
1587  initEntry(yyscanner);
1588 }
1589 //-----------------------------------------------------------------------------
1590 
1591 static void lineCount(yyscan_t yyscanner)
1592 {
1593  struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1594  DBG_CTX((stderr,"yyextra->yyLineNr=%d\n",yyextra->yyLineNr));
1595  for (const char *p = yytext; *p; ++p)
1596  {
1597  yyextra->yyLineNr += (*p == '\n') ;
1598  }
1599 }
1600 
1601 static void incLineNr(yyscan_t yyscanner)
1602 {
1603  struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1604  DBG_CTX((stderr,"yyextra->yyLineNr=%d\n",yyextra->yyLineNr));
1605  yyextra->yyLineNr++;
1606 }
1607 
1608 //-----------------------------------------------------------------
1609 static void startCommentBlock(yyscan_t yyscanner,bool brief)
1610 {
1611  struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1612  if (brief)
1613  {
1614  yyextra->current->briefFile = yyextra->yyFileName;
1615  yyextra->current->briefLine = yyextra->yyLineNr;
1616  }
1617  else
1618  {
1619  yyextra->current->docFile = yyextra->yyFileName;
1620  yyextra->current->docLine = yyextra->yyLineNr;
1621  }
1622 }
1623 
1624 static void handleCommentBlock(yyscan_t yyscanner,const QCString &doc,bool brief)
1625 {
1626  struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1627  //printf("handleCommentBlock(doc=[%s] brief=%d yyextra->docBlockInBody=%d yyextra->docBlockJavaStyle=%d\n",
1628  // qPrint(doc),brief,yyextra->docBlockInBody,yyextra->docBlockJavaStyle);
1629 
1630  // TODO: Fix me
1631  yyextra->docBlockInBody=FALSE;
1632 
1633  if (!yyextra->current->doc.isEmpty())
1634  {
1635  yyextra->current->doc=yyextra->current->doc.stripWhiteSpace()+"\n\n";
1636  }
1637  if (yyextra->docBlockInBody && yyextra->previous && !yyextra->previous->doc.isEmpty())
1638  {
1639  yyextra->previous->doc=yyextra->previous->doc.stripWhiteSpace()+"\n\n";
1640  }
1641 
1642  int position = 0;
1643  bool needsEntry = false;
1644  int lineNr = brief ? yyextra->current->briefLine : yyextra->current->docLine;
1645  Markdown markdown(yyextra->yyFileName,lineNr);
1646  QCString processedDoc = Config_getBool(MARKDOWN_SUPPORT) ? markdown.process(doc,lineNr) : doc;
1647  while (yyextra->commentScanner.parseCommentBlock(
1648  yyextra->thisParser,
1649  (yyextra->docBlockInBody && yyextra->previous) ? yyextra->previous.get() : yyextra->current.get(),
1650  processedDoc, // text
1651  yyextra->yyFileName, // file
1652  lineNr,
1653  yyextra->docBlockInBody ? FALSE : brief,
1654  yyextra->docBlockJavaStyle, // javadoc style // or FALSE,
1655  yyextra->docBlockInBody,
1656  yyextra->protection,
1657  position,
1658  needsEntry,
1659  Config_getBool(MARKDOWN_SUPPORT))
1660  ) // need to start a new entry
1661  {
1662  if (needsEntry)
1663  {
1664  newEntry(yyscanner);
1665  }
1666  }
1667  if (needsEntry)
1668  {
1669  newEntry(yyscanner);
1670  }
1671 
1672 }
1673 
1674 static void endOfDef(yyscan_t yyscanner,int correction)
1675 {
1676  struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1677  //printf("endOfDef at=%d\n",yyextra->yyLineNr);
1678  if (yyextra->bodyEntry)
1679  {
1680  yyextra->bodyEntry->endBodyLine = yyextra->yyLineNr-correction;
1681  yyextra->bodyEntry = 0;
1682  }
1683  newEntry(yyscanner);
1684  //yyextra->insideConstructor = FALSE;
1685 }
1686 
1687 static inline void addToString(yyscan_t yyscanner,const char *s)
1688 {
1689  struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1690  if (yyextra->copyString) (*yyextra->copyString) << s;
1691 }
1692 
1693 static void initTriDoubleQuoteBlock(yyscan_t yyscanner)
1694 {
1695  struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1696  yyextra->docBlockContext = YY_START;
1697  yyextra->docBlockInBody = FALSE;
1698  yyextra->docBlockJavaStyle = TRUE;
1699  yyextra->docBlockSpecial = yytext[strlen(yytext) - 1]=='!' || !Config_getBool(PYTHON_DOCSTRING);
1700  yyextra->docBlock.resize(0);
1701  yyextra->commentIndent = yyextra->curIndent;
1702  yyextra->doubleQuote = TRUE;
1703  startCommentBlock(yyscanner,FALSE);
1704 }
1705 
1706 static void initTriSingleQuoteBlock(yyscan_t yyscanner)
1707 {
1708  struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1709  yyextra->docBlockContext = YY_START;
1710  yyextra->docBlockInBody = FALSE;
1711  yyextra->docBlockJavaStyle = TRUE;
1712  yyextra->docBlockSpecial = yytext[strlen(yytext) - 1]=='!' || !Config_getBool(PYTHON_DOCSTRING);
1713  yyextra->docBlock.resize(0);
1714  yyextra->commentIndent = yyextra->curIndent;
1715  yyextra->doubleQuote = FALSE;
1716  startCommentBlock(yyscanner,FALSE);
1717 }
1718 
1719 static void initSpecialBlock(yyscan_t yyscanner)
1720 {
1721  struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1722  yyextra->docBlockContext = YY_START;
1723  yyextra->docBlockInBody = FALSE;
1724  yyextra->docBlockJavaStyle = TRUE;
1725  yyextra->docBrief = TRUE;
1726  yyextra->docBlock.resize(0);
1727  yyextra->commentIndent = yyextra->curIndent;
1728  startCommentBlock(yyscanner,FALSE);
1729 }
1730 
1731 static void searchFoundDef(yyscan_t yyscanner)
1732 {
1733  struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1734  yyextra->current->fileName = yyextra->yyFileName;
1735  yyextra->current->startLine = yyextra->yyLineNr;
1736  yyextra->current->bodyLine = yyextra->yyLineNr;
1737  yyextra->current->section = Entry::FUNCTION_SEC;
1738  yyextra->current->lang = SrcLangExt_Python;
1739  yyextra->current->virt = Normal;
1740  yyextra->current->stat = yyextra->stat;
1741  yyextra->current->mtype = yyextra->mtype = Method;
1742  yyextra->current->type.resize(0);
1743  yyextra->current->name.resize(0);
1744  yyextra->current->args.resize(0);
1745  yyextra->current->argList.clear();
1746  yyextra->packageCommentAllowed = FALSE;
1747  yyextra->stat=FALSE;
1748  //printf("searchFoundDef at=%d\n",yyextra->yyLineNr);
1749 }
1750 
1751 static void searchFoundClass(yyscan_t yyscanner)
1752 {
1753  struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1754  yyextra->current->section = Entry::CLASS_SEC;
1755  yyextra->current->argList.clear();
1756  yyextra->current->type += "class" ;
1757  yyextra->current->fileName = yyextra->yyFileName;
1758  yyextra->current->startLine = yyextra->yyLineNr;
1759  yyextra->current->bodyLine = yyextra->yyLineNr;
1760  yyextra->packageCommentAllowed = FALSE;
1761 }
1762 
1763 //----------------------------------------------------------------------------
1764 
1765 static void parseCompounds(yyscan_t yyscanner,std::shared_ptr<Entry> rt)
1766 {
1767  struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1768  //printf("parseCompounds(%s)\n",qPrint(rt->name));
1769  for (size_t i=0; i<rt->children().size(); ++i)
1770  {
1771  std::shared_ptr<Entry> ce = rt->children()[i];
1772  if (!ce->program.empty())
1773  {
1774  //fprintf(stderr,"parseCompounds: -- %s (line %d) ---------\n%s\n---------------\n",
1775  // qPrint(ce->name), ce->bodyLine, qPrint(ce->program));
1776  // init scanner state
1777  yyextra->programStr = ce->program.str();
1778  yyextra->inputString = yyextra->programStr.data();
1779  yyextra->inputPosition = 0;
1780  pyscannerYYrestart( 0, yyscanner );
1781  if (ce->section&Entry::COMPOUND_MASK)
1782  {
1783  yyextra->current_root = ce;
1784  BEGIN( Search );
1785  }
1786  else if (ce->parent())
1787  {
1788  yyextra->current_root = rt;
1789  //printf("Searching for member variables in %s parent=%s\n",
1790  // qPrint(ce->name),qPrint(ce->parent->name));
1791  BEGIN( SearchMemVars );
1792  }
1793  yyextra->yyFileName = ce->fileName;
1794  yyextra->yyLineNr = ce->bodyLine ;
1795  yyextra->current = std::make_shared<Entry>();
1796  initEntry(yyscanner);
1797 
1798  QCString name = ce->name;
1799  yyextra->commentScanner.enterCompound(yyextra->yyFileName,yyextra->yyLineNr,name);
1800 
1801  pyscannerYYlex(yyscanner) ;
1802  yyextra->lexInit=TRUE;
1803 
1804  yyextra->programStr.resize(0);
1805  ce->program.str(std::string());
1806 
1807  yyextra->commentScanner.leaveCompound(yyextra->yyFileName,yyextra->yyLineNr,name);
1808 
1809  }
1810  parseCompounds(yyscanner,ce);
1811  }
1812 }
1813 
1814 //----------------------------------------------------------------------------
1815 
1816 
1817 static void parseMain(yyscan_t yyscanner, const QCString &fileName,const char *fileBuf,const std::shared_ptr<Entry> &rt)
1818 {
1819  struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1820  initParser(yyscanner);
1821 
1822  if (fileBuf==0 || fileBuf[0]=='\0') return;
1823 
1824  yyextra->inputString = fileBuf;
1825  yyextra->inputPosition = 0;
1826 
1827  yyextra->protection = Public;
1828  yyextra->mtype = Method;
1829  yyextra->stat = FALSE;
1830  yyextra->virt = Normal;
1831  yyextra->current_root = rt;
1832  yyextra->specialBlock = FALSE;
1833 
1834  yyextra->yyLineNr= 1 ;
1835  yyextra->yyFileName = fileName;
1836  //setContext();
1837  msg("Parsing file %s...\n",qPrint(yyextra->yyFileName));
1838 
1839  FileInfo fi(fileName.str());
1840  yyextra->moduleScope = findPackageScope(yyscanner,fileName);
1841  QCString baseName=fi.baseName();
1842  if (baseName!="__init__") // package initializer file is not a package itself
1843  {
1844  if (!yyextra->moduleScope.isEmpty())
1845  {
1846  yyextra->moduleScope+="::";
1847  }
1848  yyextra->moduleScope+=baseName;
1849  }
1850 
1851  // add namespaces for each scope
1852  QCString scope = yyextra->moduleScope;
1853  int startPos = 0;
1854  int pos;
1855  do
1856  {
1857  pos = scope.find("::",startPos);
1858  startPos=pos+2;
1859  if (pos==-1) pos=(int)scope.length();
1860  yyextra->current = std::make_shared<Entry>();
1861  initEntry(yyscanner);
1862  yyextra->current->name = scope.left(pos);
1863  yyextra->current->section = Entry::NAMESPACE_SEC;
1864  yyextra->current->type = "namespace";
1865  yyextra->current->fileName = yyextra->yyFileName;
1866  yyextra->current->startLine = yyextra->yyLineNr;
1867  yyextra->current->bodyLine = yyextra->yyLineNr;
1868  yyextra->current_root = yyextra->current;
1869  rt->moveToSubEntryAndRefresh(yyextra->current);
1870  } while (pos<(int)scope.length());
1871 
1872  initParser(yyscanner);
1873 
1874  yyextra->commentScanner.enterFile(yyextra->yyFileName,yyextra->yyLineNr);
1875 
1876  yyextra->current->reset();
1877  initEntry(yyscanner);
1878  pyscannerYYrestart(0,yyscanner);
1879  BEGIN( Search );
1880  pyscannerYYlex(yyscanner);
1881  yyextra->lexInit=TRUE;
1882 
1883  yyextra->commentScanner.leaveFile(yyextra->yyFileName,yyextra->yyLineNr);
1884 
1885  yyextra->programStr.resize(0);
1886  yyextra->current_root->program.str(std::string());
1887 
1888  parseCompounds(yyscanner, yyextra->current_root);
1889 }
1890 
1891 //----------------------------------------------------------------------------
1892 
1893 static void parsePrototype(yyscan_t yyscanner,const QCString &text)
1894 {
1895  struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1896  //printf("**** parsePrototype(%s) begin\n",qPrint(text));
1897  if (text.isEmpty())
1898  {
1899  warn(yyextra->yyFileName,yyextra->yyLineNr,"Empty prototype found!");
1900  return;
1901  }
1902 
1903  yyextra->specialBlock = FALSE;
1904  yyextra->packageCommentAllowed = FALSE;
1905 
1906  const char *orgInputString;
1907  yy_size_t orgInputPosition;
1908  YY_BUFFER_STATE orgState;
1909 
1910  // save scanner state
1911  orgState = YY_CURRENT_BUFFER;
1912  yy_switch_to_buffer(yy_create_buffer(0, YY_BUF_SIZE, yyscanner), yyscanner);
1913  orgInputString = yyextra->inputString;
1914  orgInputPosition = yyextra->inputPosition;
1915 
1916  // set new string
1917  yyextra->inputString = text.data();
1918  yyextra->inputPosition = 0;
1919  pyscannerYYrestart( 0, yyscanner );
1920 
1921  BEGIN( FunctionDec );
1922 
1923  pyscannerYYlex(yyscanner);
1924  yyextra->lexInit=TRUE;
1925 
1926  yyextra->current->name = yyextra->current->name.stripWhiteSpace();
1927  if (yyextra->current->section == Entry::MEMBERDOC_SEC && yyextra->current->args.isEmpty())
1928  yyextra->current->section = Entry::VARIABLEDOC_SEC;
1929 
1930  // restore original scanner state
1931 
1932  yy_delete_buffer(YY_CURRENT_BUFFER, yyscanner);
1933  yy_switch_to_buffer(orgState, yyscanner);
1934 
1935  yyextra->inputString = orgInputString;
1936  yyextra->inputPosition = orgInputPosition;
1937 
1938  //printf("**** parsePrototype end\n");
1939 }
1940 
1941 //----------------------------------------------------------------------------
1942 
1943 struct PythonOutlineParser::Private
1944 {
1945  yyscan_t yyscanner;
1946  pyscannerYY_state state;
1947 };
1948 
1949 PythonOutlineParser::PythonOutlineParser() : p(std::make_unique<PythonOutlineParser::Private>())
1950 {
1951  pyscannerYYlex_init_extra(&p->state,&p->yyscanner);
1952 #ifdef FLEX_DEBUG
1953  pyscannerYYset_debug(1,p->yyscanner);
1954 #endif
1955 }
1956 
1957 PythonOutlineParser::~PythonOutlineParser()
1958 {
1959  pyscannerYYlex_destroy(p->yyscanner);
1960 }
1961 
1962 
1963 void PythonOutlineParser::parseInput(const QCString &fileName,
1964  const char *fileBuf,
1965  const std::shared_ptr<Entry> &root,
1966  ClangTUParser * /*clangParser*/)
1967 {
1968  struct yyguts_t *yyg = (struct yyguts_t*)p->yyscanner;
1969  yyextra->thisParser = this;
1970  printlex(yy_flex_debug, TRUE, __FILE__, qPrint(fileName));
1971  ::parseMain(p->yyscanner, fileName,fileBuf,root);
1972  printlex(yy_flex_debug, FALSE, __FILE__, qPrint(fileName));
1973 
1974  // May print the AST for debugging purposes
1975  // printAST(global_root);
1976 }
1977 
1978 bool PythonOutlineParser::needsPreprocessing(const QCString &) const
1979 {
1980  return FALSE;
1981 }
1982 
1983 void PythonOutlineParser::parsePrototype(const QCString &text)
1984 {
1985  ::parsePrototype(p->yyscanner,text);
1986 }
1987 
1988 //----------------------------------------------------------------------------
1989 
1990 #if USE_STATE2STRING
1991 #include "pyscanner.l.h"
1992 #endif