Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Account for line endings in logging #430

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
mrunalp wants to merge 2 commits into from
Closed

Account for line endings in logging #430

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
eb39b4a
Account for line endings in logging
mrunalp Apr 5, 2017
cbbf9ae
Don't write \0 after timestamp and skip line ending on partial lines
mrunalp Apr 7, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 65 additions & 22 deletions conmon/conmon.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,68 @@ int set_k8s_timestamp(char *buf, ssize_t buflen, const char *stream_type)
return 0;
}


/*
* splits buf into lines and inserts timestamps at the beginning of
* the line written to logfd.
*/
int write_with_timestamps(int logfd, const char *buf, ssize_t buflen)
{
#define TSBUFLEN 34
char tsbuf[TSBUFLEN];
static bool last_buf_ended_with_newline = TRUE;

g_auto(GStrv) lines = g_strsplit(buf, "\n", -1);
ssize_t num_lines = g_strv_length(lines);
for (ssize_t i = 0; i < num_lines; i++)
{
const char *line = lines[i];

ninfo("Processing line: %ld, %s", i, line);

/* Skip last line if it is empty */
if (i == (num_lines - 1) && buf[buflen-1] == '\n' && !strcmp("", line)) {
ninfo("Skipping last line");
break;
}

/*
* Only add timestamps for first line if last buffer's last
* line ended with newline. Add it for all other lines.
*/
if (i != 0 || (i == 0 && last_buf_ended_with_newline)) {
ninfo("Adding timestamp");
int rc = set_k8s_timestamp(tsbuf, TSBUFLEN, "stdout");
Copy link
Member

@runcom runcom Apr 6, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Removing the \0 added at line 486 from my comment above, I went testing this again:

snprintf in set_k8s_timestamp is adding a \0 char at the end of the buf. When the log is read by k8s the hex dump is as follow:

002f62696e2f73683a2063616e277420637265617465202f66696c653a20526561642d6f6e6c792066696c652073797374656d0a

notice the 00 at the beginning. Which makes this test fail with:

    it should not write to root filesystem [Conformance] [It]
    /home/amurdaca/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/test/e2e_node/kubelet_test.go:154

    Timed out after 60.000s.
    Expected
        <string>: /bin/sh: can't create /file: Read-only file system

    to equal
        <string>: /bin/sh: can't create /file: Read-only file system

even if strings appear to be the same.

The thing should be that, when we write the timestamp line with the log, we should snprintf with the whole log line and remove the NULL byte at the end before writeing, this patch does that but it's ugly I know :):

diff --git a/conmon/conmon.c b/conmon/conmon.c
index f69720f..8c4fdc7 100644
--- a/conmon/conmon.c
+++ b/conmon/conmon.c
@@ -93,7 +93,7 @@ static GOptionEntry entries[] =
   { NULL }
 };
 
-int set_k8s_timestamp(char *buf, ssize_t buflen, const char *stream_type)
+int set_k8s_timestamp(int logfd, char *buf, ssize_t buflen, const char *stream_type, const char *log)
 {
 	time_t now = time(NULL);
 	struct tm *tm;
@@ -108,10 +108,17 @@ int set_k8s_timestamp(char *buf, ssize_t buflen, const char *stream_type)
 		off_sign = '-';
 		off = -off;
 	}
-	snprintf(buf, buflen, "%d-%02d-%02dT%02d:%02d:%02d%c%02d:%02d %s ",
+	snprintf(buf, buflen, "%d-%02d-%02dT%02d:%02d:%02d%c%02d:%02d %s %s",
 		tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday,
 		tm->tm_hour, tm->tm_min, tm->tm_sec,
-		off_sign, off / 3600, off % 3600, stream_type);
+		off_sign, off / 3600, off % 3600, stream_type, log);
+
+	char newbuf[buflen-1];
+	strncpy(newbuf, buf, buflen-1);
+
+	if (write(logfd, newbuf, buflen-1) != buflen-1) {
+		nwarn("partial/failed write ts (logFd)");
+	}
 
 	return 0;
 }
@@ -124,7 +131,6 @@ int set_k8s_timestamp(char *buf, ssize_t buflen, const char *stream_type)
 int write_with_timestamps(int logfd, const char *buf, ssize_t buflen)
 {
 	#define TSBUFLEN 34
-	char tsbuf[TSBUFLEN];
 	static bool last_buf_ended_with_newline = TRUE;
 
 	g_auto(GStrv) lines = g_strsplit(buf, "\n", -1);
@@ -147,21 +153,24 @@ int write_with_timestamps(int logfd, const char *buf, ssize_t buflen)
 		 */
 		if (i != 0 || (i == 0 && last_buf_ended_with_newline)) {
 			ninfo("Adding timestamp");
-			int rc = set_k8s_timestamp(tsbuf, TSBUFLEN, "stdout");
+			ssize_t len = strlen(line);
+			char tsbuf[TSBUFLEN+len];
+			ninfo("fuck %ld", TSBUFLEN+len);
+			int rc = set_k8s_timestamp(logfd, tsbuf, TSBUFLEN+len, "stdout", line);
 			if (rc < 0) {
 				nwarn("failed to set timestamp");
 			} else {
-				if (write(logfd, tsbuf, TSBUFLEN) != TSBUFLEN) {
-					nwarn("partial/failed write ts (logFd)");
-				}
+				/*if (write(logfd, tsbuf, TSBUFLEN+len) != TSBUFLEN+len) {*/
+					/*nwarn("partial/failed write ts (logFd)");*/
+				/*}*/
+			}
+		} else {
+			/* Log output to logfd. */
+			ssize_t len = strlen(line);
+			if (write(logfd, line, len) != len) {
+				nwarn("partial/failed write (logFd)");
+				return -1;
 			}
-		}
-
-		/* Log output to logfd. */
-		ssize_t len = strlen(line);
-		if (write(logfd, line, len) != len) {
-			nwarn("partial/failed write (logFd)");
-			return -1;
 		}
 		/* Write the line ending */
 		if (write(logfd, "\n", 1) != 1) {
@@ -483,7 +492,7 @@ int main(int argc, char *argv[])
 					if (num_read <= 0)
 						goto out;
 
-					buf[num_read] = '\0';
+					/*buf[num_read] = '\0';*/
 					ninfo("read a chunk: (fd=%d) '%s'", mfd, buf);
 
 					/* Insert CRI mandated timestamps in the buffer for each line */

This patch makes the test above pass. The patch is ugly but the thing is we should not play with null-terminated strings when printing to logfd, or when read back we have issues.

Copy link
Member

@runcom runcom Apr 6, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Again, the patch is ugly and I'm by any mean a good C code writer so if you guys can come up with a better solution please discard my whole comment :)

Copy link
Contributor

@cyphar cyphar Apr 6, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@runcom ninfo("fuck %ld", TSBUFLEN+len); sort of captures my opinion about having to do this in C 😉. I think we could make this code "nicer" if we wrote partial lines to the log (so we just append the timestamp after the \n and don't do any of this code that stores the "remaining" part of the line). The downside is that a user might be able to see the partially-written lines. I don't think it should happen that often in practice, but you never know.

If you want, I'll open up a PR with my implementation of this so you can compare it with this.

Copy link
Member Author

@mrunalp mrunalp Apr 6, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We are taking care of partial lines in the PR. The logic will always have edge cases. For e.g. if we append after \n instead of inserting at beginning we still need to add one at the beginning of the log buffer. In my approach I am tracking if previous buffer ended with \n or not and adding a timestamp accordingly. One thing that is missing and needs to be handled is when there are no \n in a buf.

Copy link
Contributor

@cyphar cyphar Apr 6, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah right, I misread the code. In any case, #436 is my implementation.

if (rc < 0) {
nwarn("failed to set timestamp");
} else {
/* Exclude the \0 while writing */
if (write(logfd, tsbuf, TSBUFLEN - 1) != (TSBUFLEN - 1)) {
nwarn("partial/failed write ts (logFd)");
}
}
}

/* Log output to logfd. */
ssize_t len = strlen(line);
if (write(logfd, line, len) != len) {
nwarn("partial/failed write (logFd)");
return -1;
}
/* Write the line ending */
if ((i < num_lines - 1) || (i == (num_lines - 1) && buf[buflen - 1] == '\n')) {
if (write(logfd, "\n", 1) != 1) {
nwarn("failed to write line ending");
return -1;
}
}
}

last_buf_ended_with_newline = buf[buflen-1] == '\n';

return 0;
}

int main(int argc, char *argv[])
{
int ret, runtime_status;
Expand Down Expand Up @@ -395,9 +457,6 @@ int main(int argc, char *argv[])
pexit("Failed to add console master fd to epoll");
}

#define TSBUFLEN 34
char tsbuf[TSBUFLEN];

/*
* Log all of the container's output and pipe STDIN into it. Currently
* nothing using the STDIN setup (which makes its inclusion here a bit
Expand Down Expand Up @@ -427,28 +486,12 @@ int main(int argc, char *argv[])
if (num_read <= 0)
goto out;

buf[num_read] = '\0';
Copy link
Member

@runcom runcom Apr 6, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is splitting every line into \0-separated strings like:

[conmon:i]: container PID: 7805
[conmon:i]: read a chunk: (fd=5) '/bin/sh'
[conmon:i]: Processing line: 0, /bin/sh
[conmon:i]: Adding timestamp
[conmon:i]: read a chunk: (fd=5) ': can't create /file: '
[conmon:i]: Processing line: 0, : can't create /file:
[conmon:i]: read a chunk: (fd=5) 'Read-only file system
'
[conmon:i]: Processing line: 0, Read-only file system
[conmon:i]: Processing line: 1,

and it's breaking tests (removing this reads the whole line)

ninfo("read a chunk: (fd=%d) '%s'", mfd, buf);


/* Prepend the CRI-mandated timestamp and other metadata. */
/*
* FIXME: Currently this code makes the assumption that
* @buf doesn't contain any newlines (since the CRI
* requires each line to contain a timestamp). This
* is an /okay/ assumption in most cases because
* ptys generally have newline-buffered output.
*/
int rc = set_k8s_timestamp(tsbuf, TSBUFLEN, "stdout");
/* Insert CRI mandated timestamps in the buffer for each line */
int rc = write_with_timestamps(logfd, buf, num_read);
if (rc < 0) {
nwarn("failed to set timestamp");
} else {
if (write(logfd, tsbuf, TSBUFLEN) != TSBUFLEN) {
nwarn("partial/failed write ts (logFd)");
}
}
/* Log all output to logfd. */
if (write(logfd, buf, num_read) != num_read) {
nwarn("partial/failed write (logFd)");
goto out;
}
}
Expand Down