[PATCH] support for XML files in imfile plugin

This is the place for developers to discuss bugs, new features and everything else about code changes.

Google Ads


[PATCH] support for XML files in imfile plugin

Postby prune » Tue Jun 19, 2012 7:58 pm

In latest rSyslog 6.3.x the imfile plugin allows for multiline files monitoring.
This is done using the option :
$InputFileReadMode 2

This option is working as :
# 0 = normal, 1 = blank line, 2 = indented

I added another option :
# 0 = normal, 1 = blank line, 2 = indented, 3 = XML file

Here is the patch file against the trunk (6.3.11 version) (diff -u) :

Code: Select all
--- rsyslog-6.3.11/runtime/stream.c     2012-04-12 08:11:08.000000000 -0400
+++ rsyslog/runtime/stream.c    2012-06-19 14:51:48.388283508 -0400
@@ -585,11 +585,13 @@
        /* mode = 0 single line mode (equivalent to ReadLine)
          * mode = 1 LFLF mode (paragraph, blank line between entries)
          * mode = 2 LF <not whitespace> mode, a log line starts at the beginning of a line, but following lines that are indented are part of the same log entry
+        * mode = 3 XML file, a log line start with a <tag> and end with the closing </tag>. Every tags in between are the same line
         *  This modal interface is not nearly as flexible as being able to define a regex for when a new record starts, but it's also not nearly as hard (or as slow) to implement
          */
         DEFiRet;
         uchar c;
        uchar finished;
+       uchar indented;

         ASSERT(pThis != NULL);
         ASSERT(ppCStr != NULL);
@@ -661,6 +663,76 @@
                }
                        CHKiRet(cstrFinalize(*ppCStr));
        }
+        if (mode == 3){
+               /* XML style lines */
+               /*
+                       Will match content from <tag..> to </tag>
+                       This will not work if the plugins open an unfiniched XML file
+                       Ensure your XML file is valid and closed before starting the plugins
+               */
+               finished=0;
+               indented=0;
+               while(finished == 0){
+                       if ((*ppCStr)->iStrLen == 0){
+                               if(c != '\n') {
+                               /* nothing in the buffer, and it's not a newline, add it to the buffer */
+                                       if (c == '<') {
+                                               indented++;
+                                       }
+                                               CHKiRet(cstrAppendChar(*ppCStr, c));
+                                               CHKiRet(strmReadChar(pThis, &c));
+                               } else {
+                                       finished=1;  /* this is a blank line, a \n with nothing since the last complete record */
+                               }
+                       } else {
+                               /* not the first character after a newline, add it to the buffer */
+                               if (((*ppCStr)->pBuf[(*ppCStr)->iStrLen -1 ] == '\n')&& (indented == 0)){
+                                       CHKiRet(strmUnreadChar(pThis, c));
+                                       rsCStrTruncate(*ppCStr,1);
+                                       finished=1;
+                               }
+
+                               else {
+                                       if (c == '<') {
+                                               CHKiRet(cstrAppendChar(*ppCStr, '<'));
+                                               CHKiRet(strmReadChar(pThis, &c));
+                                               if (c == '/') {
+                                                       indented--;
+                                                       // if indented goes below 0, finish the line and startover
+                                                       if (indented < 0) {
+                                                               indented=0;
+                                                               finished=1;
+                                                       }
+                                               }
+                                               else {
+                                                       indented++;
+                                               }
+
+                                       }
+                                       if ((*ppCStr)->pBuf[(*ppCStr)->iStrLen -1 ] != '\n'){
+
+                                                       CHKiRet(cstrAppendChar(*ppCStr, c));
+                                                       CHKiRet(strmReadChar(pThis, &c));
+                                       } else {
+                                               //if ((c == ' ') || (c == '\t')){
+                                               if (indented > 0) {
+                                                               CHKiRet(cstrAppendChar(*ppCStr, c));
+                                                               CHKiRet(strmReadChar(pThis, &c));
+                                               } else {
+                                                       /* clean things up by putting the character we just read back into
+                                                        * the input buffer and removing the LF character that is currently at the
+                                                        * end of the output string */
+                                                       CHKiRet(strmUnreadChar(pThis, c));
+                                                       rsCStrTruncate(*ppCStr,1);
+                                                       finished=1;
+                                               }
+                                       }
+                               }
+                       }
+               }
+                       CHKiRet(cstrFinalize(*ppCStr));
+       }
+

 finalize_it:
         if(iRet != RS_RET_OK && *ppCStr != NULL)


This will match a whole XML block as a sing syslog message.
Be warned that syslog messages are limited to 1024 bytes. rSyslog can handle them fine if longer, but not every other syslog will do (for example syslog4j wont, so GrayLog2 CAN'T store them if longer than 1024 bytes.

For exemple :

<record>
<date>2012-06-13T11:09:21</date>
<sequence>4</sequence>
<level>INFO</level>
</record>

will be stored as a single message.

What is the next step to add this to the trunk dev ? Maybe a better coder than me can have a look and correct/optimize...
Comments welcome.
This is used on production without issue for two weeks now...
prune
Avarage
 
Posts: 13
Joined: Mon Jan 23, 2012 5:09 pm

Urgent Question?

  • Pulling out your Hair?
  • Wasting Time and Money?
  • Deadline Approaching?

Re: [PATCH] support for XML files in imfile plugin

Postby prune » Wed Jun 20, 2012 8:52 pm

This patch is not working if you have an XML file "never ending"... I'm working on a new version with a config parameter to define the start tag (ex : "record").
Please do not use this patch it may not work for you.
prune
Avarage
 
Posts: 13
Joined: Mon Jan 23, 2012 5:09 pm

Google Ads



Return to Developer's Corner

Who is online

Users browsing this forum: No registered users and 0 guests

cron