Account for line endings in logging #430

mrunalp · 2017-04-05T23:05:03Z

Closes #429

Signed-off-by: Mrunal Patel [email protected]

Signed-off-by: Mrunal Patel <[email protected]>

mrunalp · 2017-04-05T23:21:19Z

If you test with yaml from #429

[root@dhcp-16-129 kubernetes]# ./cluster/kubectl.sh create -f ~/pods/env_test.yaml                                                                                                                                                                            
pod "envar-demo-4" created
[root@dhcp-16-129 kubernetes]# ./cluster/kubectl.sh logs envar-demo-4
KUBERNETES_PORT=tcp://10.0.0.1:443
KUBERNETES_SERVICE_PORT=443
FOOBAR=foo-value;;bar-value
SHLVL=1
HOME=/root
TERM=xterm
KUBERNETES_PORT_443_TCP_ADDR=10.0.0.1
BAR=bar-value
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
KUBERNETES_PORT_4
FOO=foo-value
KUBERNETES_PORT_443_TCP_PROTO=tcp
KUBERNETES_PORT_443_TCP=tcp://10.0.0.1:443
KUBERNETES_SERVICE_PORT_HTTPS=443
PWD=/
KUBERNETES_SERVICE_HOST=10.0.0.1

runcom · 2017-04-06T08:20:19Z

conmon build failures (though, on Fedora, it compiles just fine)

runcom · 2017-04-06T08:56:49Z

conmon/conmon.c

 					if (num_read <= 0)
 						goto out;

+					buf[num_read] = '\0';


this is splitting every line into \0-separated strings like:

[conmon:i]: container PID: 7805 [conmon:i]: read a chunk: (fd=5) '/bin/sh' [conmon:i]: Processing line: 0, /bin/sh [conmon:i]: Adding timestamp [conmon:i]: read a chunk: (fd=5) ': can't create /file: ' [conmon:i]: Processing line: 0, : can't create /file: [conmon:i]: read a chunk: (fd=5) 'Read-only file system ' [conmon:i]: Processing line: 0, Read-only file system [conmon:i]: Processing line: 1,

and it's breaking tests (removing this reads the whole line)

runcom · 2017-04-06T09:57:41Z

conmon/conmon.c

+		 */
+		if (i != 0 || (i == 0 && last_buf_ended_with_newline)) {
+			ninfo("Adding timestamp");
+			int rc = set_k8s_timestamp(tsbuf, TSBUFLEN, "stdout");


Removing the \0 added at line 486 from my comment above, I went testing this again:

snprintf in set_k8s_timestamp is adding a \0 char at the end of the buf. When the log is read by k8s the hex dump is as follow:

002f62696e2f73683a2063616e277420637265617465202f66696c653a20526561642d6f6e6c792066696c652073797374656d0a

notice the 00 at the beginning. Which makes this test fail with:

it should not write to root filesystem [Conformance] [It] /home/amurdaca/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/test/e2e_node/kubelet_test.go:154 Timed out after 60.000s. Expected <string>: /bin/sh: can't create /file: Read-only file system to equal <string>: /bin/sh: can't create /file: Read-only file system

even if strings appear to be the same.

The thing should be that, when we write the timestamp line with the log, we should snprintf with the whole log line and remove the NULL byte at the end before writeing, this patch does that but it's ugly I know :):

diff --git a/conmon/conmon.c b/conmon/conmon.c index f69720f..8c4fdc7 100644 --- a/conmon/conmon.c +++ b/conmon/conmon.c @@ -93,7 +93,7 @@ static GOptionEntry entries[] = { NULL } }; -int set_k8s_timestamp(char *buf, ssize_t buflen, const char *stream_type) +int set_k8s_timestamp(int logfd, char *buf, ssize_t buflen, const char *stream_type, const char *log) { time_t now = time(NULL); struct tm *tm; @@ -108,10 +108,17 @@ int set_k8s_timestamp(char *buf, ssize_t buflen, const char *stream_type) off_sign = '-'; off = -off; } - snprintf(buf, buflen, "%d-%02d-%02dT%02d:%02d:%02d%c%02d:%02d %s ", + snprintf(buf, buflen, "%d-%02d-%02dT%02d:%02d:%02d%c%02d:%02d %s %s", tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec, - off_sign, off / 3600, off % 3600, stream_type); + off_sign, off / 3600, off % 3600, stream_type, log); + + char newbuf[buflen-1]; + strncpy(newbuf, buf, buflen-1); + + if (write(logfd, newbuf, buflen-1) != buflen-1) { + nwarn("partial/failed write ts (logFd)"); + } return 0; } @@ -124,7 +131,6 @@ int set_k8s_timestamp(char *buf, ssize_t buflen, const char *stream_type) int write_with_timestamps(int logfd, const char *buf, ssize_t buflen) { #define TSBUFLEN 34 - char tsbuf[TSBUFLEN]; static bool last_buf_ended_with_newline = TRUE; g_auto(GStrv) lines = g_strsplit(buf, "\n", -1); @@ -147,21 +153,24 @@ int write_with_timestamps(int logfd, const char *buf, ssize_t buflen) */ if (i != 0 || (i == 0 && last_buf_ended_with_newline)) { ninfo("Adding timestamp"); - int rc = set_k8s_timestamp(tsbuf, TSBUFLEN, "stdout"); + ssize_t len = strlen(line); + char tsbuf[TSBUFLEN+len]; + ninfo("fuck %ld", TSBUFLEN+len); + int rc = set_k8s_timestamp(logfd, tsbuf, TSBUFLEN+len, "stdout", line); if (rc < 0) { nwarn("failed to set timestamp"); } else { - if (write(logfd, tsbuf, TSBUFLEN) != TSBUFLEN) { - nwarn("partial/failed write ts (logFd)"); - } + /*if (write(logfd, tsbuf, TSBUFLEN+len) != TSBUFLEN+len) {*/ + /*nwarn("partial/failed write ts (logFd)");*/ + /*}*/ + } + } else { + /* Log output to logfd. */ + ssize_t len = strlen(line); + if (write(logfd, line, len) != len) { + nwarn("partial/failed write (logFd)"); + return -1; } - } - - /* Log output to logfd. */ - ssize_t len = strlen(line); - if (write(logfd, line, len) != len) { - nwarn("partial/failed write (logFd)"); - return -1; } /* Write the line ending */ if (write(logfd, "\n", 1) != 1) { @@ -483,7 +492,7 @@ int main(int argc, char *argv[]) if (num_read <= 0) goto out; - buf[num_read] = '\0'; + /*buf[num_read] = '\0';*/ ninfo("read a chunk: (fd=%d) '%s'", mfd, buf); /* Insert CRI mandated timestamps in the buffer for each line */

This patch makes the test above pass. The patch is ugly but the thing is we should not play with null-terminated strings when printing to logfd, or when read back we have issues.

Again, the patch is ugly and I'm by any mean a good C code writer so if you guys can come up with a better solution please discard my whole comment :)

@runcom ninfo("fuck %ld", TSBUFLEN+len); sort of captures my opinion about having to do this in C 😉. I think we could make this code "nicer" if we wrote partial lines to the log (so we just append the timestamp after the \n and don't do any of this code that stores the "remaining" part of the line). The downside is that a user might be able to see the partially-written lines. I don't think it should happen that often in practice, but you never know.

If you want, I'll open up a PR with my implementation of this so you can compare it with this.

We are taking care of partial lines in the PR. The logic will always have edge cases. For e.g. if we append after \n instead of inserting at beginning we still need to add one at the beginning of the log buffer. In my approach I am tracking if previous buffer ended with \n or not and adding a timestamp accordingly. One thing that is missing and needs to be handled is when there are no \n in a buf.

Ah right, I misread the code. In any case, #436 is my implementation.

runcom · 2017-04-06T13:42:18Z

Well, my patch + @mrunalp's changes in this PR are raising tests percentage of success a lot 😄

cyphar · 2017-04-06T17:23:58Z

I'll write an alternative implementation to this to see whether we can make this code somewhat bearable, though I think there isn't a nice way of implementing this in C.

mrunalp · 2017-04-06T17:25:50Z

@cyphar Sure, see if you can come with something cleaner :)

cyphar · 2017-04-06T17:32:12Z

I almost immediately realised that my idea for partial-line-writing won't work if we have multiple fds for different stdio streams. Luckily no container runtime actually implements that properly (yet) so we can punt on that issue for the moment.

runcom · 2017-04-06T18:04:34Z

@mrunalp @cyphar cool, so, should I go and fix Mrunal's comment in mrunalp#1 or just wait for Aleksa's alternate implementation?

Signed-off-by: Antonia Murdaca <[email protected]> Signed-off-by: Mrunal Patel <[email protected]>

runcom · 2017-04-11T11:31:55Z

@mrunalp should we close this one in favor of #436 ?

cyphar · 2017-04-11T16:00:55Z

#436 has been merged.

k8s-ci-robot added the cncf-cla: yes label Apr 5, 2017

Account for line endings in logging

eb39b4a

Signed-off-by: Mrunal Patel <[email protected]>

mrunalp force-pushed the logging/line_endings branch from cc7f944 to eb39b4a Compare April 5, 2017 23:18

mrunalp added this to the 0.2 milestone Apr 5, 2017

runcom reviewed Apr 6, 2017

View reviewed changes

cyphar mentioned this pull request Apr 6, 2017

conmon: handle multi-line logging #436

Merged

runcom mentioned this pull request Apr 7, 2017

conmon: do not put NULLs in log lines mrunalp/cri-o#1

Closed

mrunalp force-pushed the logging/line_endings branch from 87b7b58 to b6e5f2e Compare April 7, 2017 17:52

Don't write \0 after timestamp and skip line ending on partial lines

cbbf9ae

Signed-off-by: Antonia Murdaca <[email protected]> Signed-off-by: Mrunal Patel <[email protected]>

mrunalp force-pushed the logging/line_endings branch from b6e5f2e to cbbf9ae Compare April 7, 2017 17:55

cyphar closed this Apr 11, 2017

Account for line endings in logging #430

Account for line endings in logging #430

Uh oh!

Conversation

mrunalp commented Apr 5, 2017

Uh oh!

mrunalp commented Apr 5, 2017

Uh oh!

runcom commented Apr 6, 2017 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

runcom Apr 6, 2017

Choose a reason for hiding this comment

Uh oh!

runcom Apr 6, 2017 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Choose a reason for hiding this comment

Uh oh!

runcom Apr 6, 2017

Choose a reason for hiding this comment

Uh oh!

cyphar Apr 6, 2017

Choose a reason for hiding this comment

Uh oh!

mrunalp Apr 6, 2017

Choose a reason for hiding this comment

Uh oh!

cyphar Apr 6, 2017

Choose a reason for hiding this comment

Uh oh!

runcom commented Apr 6, 2017

Uh oh!

cyphar commented Apr 6, 2017 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

mrunalp commented Apr 6, 2017

Uh oh!

cyphar commented Apr 6, 2017 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

runcom commented Apr 6, 2017

Uh oh!

runcom commented Apr 11, 2017

Uh oh!

cyphar commented Apr 11, 2017

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

4 participants

runcom commented Apr 6, 2017 •

edited

Loading

runcom Apr 6, 2017 •

edited

Loading

cyphar commented Apr 6, 2017 •

edited

Loading

cyphar commented Apr 6, 2017 •

edited

Loading