| #include <assert.h> |
| #include <ctype.h> |
| #include <fcntl.h> |
| #include <stdio.h> |
| #include <string.h> |
| #include <sys/mman.h> |
| #include <sys/stat.h> |
| #include <sys/types.h> |
| #include <unistd.h> |
| |
| #include <json.h> |
| |
| #include "igt_core.h" |
| #include "resultgen.h" |
| #include "settings.h" |
| #include "executor.h" |
| #include "output_strings.h" |
| |
| #define INCOMPLETE_EXITCODE -1 |
| |
| _Static_assert(INCOMPLETE_EXITCODE != IGT_EXIT_SKIP, "exit code clash"); |
| _Static_assert(INCOMPLETE_EXITCODE != IGT_EXIT_SUCCESS, "exit code clash"); |
| _Static_assert(INCOMPLETE_EXITCODE != IGT_EXIT_INVALID, "exit code clash"); |
| |
| struct subtests |
| { |
| char **names; |
| size_t size; |
| }; |
| |
| struct results |
| { |
| struct json_object *tests; |
| struct json_object *totals; |
| struct json_object *runtimes; |
| }; |
| |
| /* |
| * A lot of string handling here operates on an mmapped buffer, and |
| * thus we can't assume null-terminated strings. Buffers will be |
| * passed around as pointer+size, or pointer+pointer-past-the-end, the |
| * mem*() family of functions is used instead of str*(). |
| */ |
| |
| static char *find_line_starting_with(char *haystack, const char *needle, char *end) |
| { |
| while (haystack < end) { |
| char *line_end = memchr(haystack, '\n', end - haystack); |
| |
| if (end - haystack < strlen(needle)) |
| return NULL; |
| if (!memcmp(haystack, needle, strlen(needle))) |
| return haystack; |
| if (line_end == NULL) |
| return NULL; |
| haystack = line_end + 1; |
| } |
| |
| return NULL; |
| } |
| |
| static char *find_line_starting_with_either(char *haystack, |
| const char *needle1, |
| const char *needle2, |
| char *end) |
| { |
| while (haystack < end) { |
| char *line_end = memchr(haystack, '\n', end - haystack); |
| size_t linelen = line_end != NULL ? line_end - haystack : end - haystack; |
| if ((linelen >= strlen(needle1) && !memcmp(haystack, needle1, strlen(needle1))) || |
| (linelen >= strlen(needle2) && !memcmp(haystack, needle2, strlen(needle2)))) |
| return haystack; |
| |
| if (line_end == NULL) |
| return NULL; |
| |
| haystack = line_end + 1; |
| } |
| |
| return NULL; |
| } |
| |
| static char *next_line(char *line, char *bufend) |
| { |
| char *ret; |
| |
| if (!line) |
| return NULL; |
| |
| ret = memchr(line, '\n', bufend - line); |
| if (ret) |
| ret++; |
| |
| if (ret < bufend) |
| return ret; |
| else |
| return NULL; |
| } |
| |
| static char *find_line_after_last(char *begin, |
| const char *needle1, |
| const char *needle2, |
| char *end) |
| { |
| char *one, *two; |
| char *current_pos = begin; |
| char *needle1_newline = malloc(strlen(needle1) + 2); |
| char *needle2_newline = malloc(strlen(needle2) + 2); |
| |
| needle1_newline[0] = needle2_newline[0] = '\n'; |
| strcpy(needle1_newline + 1, needle1); |
| strcpy(needle2_newline + 1, needle2); |
| |
| while (true) { |
| one = memmem(current_pos, end - current_pos, needle1_newline, strlen(needle1_newline)); |
| two = memmem(current_pos, end - current_pos, needle2_newline, strlen(needle2_newline)); |
| if (one == NULL && two == NULL) |
| break; |
| |
| if (one != NULL && current_pos < one) |
| current_pos = one; |
| if (two != NULL && current_pos < two) |
| current_pos = two; |
| |
| one = next_line(current_pos, end); |
| if (one != NULL) |
| current_pos = one; |
| } |
| free(needle1_newline); |
| free(needle2_newline); |
| |
| one = memchr(current_pos, '\n', end - current_pos); |
| if (one != NULL) |
| return ++one; |
| |
| return current_pos; |
| } |
| |
| static size_t count_lines(const char *buf, const char *bufend) |
| { |
| size_t ret = 0; |
| while (buf < bufend && (buf = memchr(buf, '\n', bufend - buf)) != NULL) { |
| ret++; |
| buf++; |
| } |
| |
| return ret; |
| } |
| |
| static void append_line(char **buf, size_t *buflen, char *line) |
| { |
| size_t linelen = strlen(line); |
| |
| *buf = realloc(*buf, *buflen + linelen + 1); |
| strcpy(*buf + *buflen, line); |
| *buflen += linelen; |
| } |
| |
| static const struct { |
| const char *output_str; |
| const char *result_str; |
| } resultmap[] = { |
| { "SUCCESS", "pass" }, |
| { "SKIP", "skip" }, |
| { "FAIL", "fail" }, |
| { "CRASH", "crash" }, |
| { "TIMEOUT", "timeout" }, |
| }; |
| static void parse_result_string(char *resultstring, size_t len, const char **result, double *time) |
| { |
| size_t i; |
| size_t wordlen = 0; |
| |
| while (wordlen < len && !isspace(resultstring[wordlen])) { |
| wordlen++; |
| } |
| |
| *result = NULL; |
| for (i = 0; i < (sizeof(resultmap) / sizeof(resultmap[0])); i++) { |
| if (!strncmp(resultstring, resultmap[i].output_str, wordlen)) { |
| *result = resultmap[i].result_str; |
| break; |
| } |
| } |
| |
| /* If the result string is unknown, use incomplete */ |
| if (!*result) |
| *result = "incomplete"; |
| |
| /* |
| * Check for subtest runtime after the result. The string is |
| * '(' followed by the runtime in seconds as floating point, |
| * followed by 's)'. |
| */ |
| wordlen++; |
| if (wordlen < len && resultstring[wordlen] == '(') { |
| char *dup; |
| |
| wordlen++; |
| dup = malloc(len - wordlen + 1); |
| memcpy(dup, resultstring + wordlen, len - wordlen); |
| dup[len - wordlen] = '\0'; |
| *time = strtod(dup, NULL); |
| |
| free(dup); |
| } |
| } |
| |
| static void parse_subtest_result(char *subtest, const char **result, double *time, char *buf, char *bufend) |
| { |
| char *line; |
| char *line_end; |
| char *resultstring; |
| size_t linelen; |
| size_t subtestlen = strlen(subtest); |
| |
| *result = NULL; |
| *time = 0.0; |
| |
| if (!buf) return; |
| |
| /* |
| * The result line structure is: |
| * |
| * - The string "Subtest " (`SUBTEST_RESULT` from output_strings.h) |
| * - The subtest name |
| * - The characters ':' and ' ' |
| * - Subtest result string |
| * - Optional: |
| * -- The characters ' ' and '(' |
| * -- Subtest runtime in seconds as floating point |
| * -- The characters 's' and ')' |
| * |
| * Example: |
| * Subtest subtestname: PASS (0.003s) |
| */ |
| |
| line = find_line_starting_with(buf, SUBTEST_RESULT, bufend); |
| if (!line) { |
| *result = "incomplete"; |
| return; |
| } |
| |
| line_end = memchr(line, '\n', bufend - line); |
| linelen = line_end != NULL ? line_end - line : bufend - line; |
| |
| if (strlen(SUBTEST_RESULT) + subtestlen + strlen(": ") > linelen || |
| strncmp(line + strlen(SUBTEST_RESULT), subtest, subtestlen)) |
| return parse_subtest_result(subtest, result, time, line + linelen, bufend); |
| |
| resultstring = line + strlen(SUBTEST_RESULT) + subtestlen + strlen(": "); |
| parse_result_string(resultstring, linelen - (resultstring - line), result, time); |
| } |
| |
| static struct json_object *get_or_create_json_object(struct json_object *base, |
| const char *key) |
| { |
| struct json_object *ret; |
| |
| if (json_object_object_get_ex(base, key, &ret)) |
| return ret; |
| |
| ret = json_object_new_object(); |
| json_object_object_add(base, key, ret); |
| |
| return ret; |
| } |
| |
| static void set_result(struct json_object *obj, const char *result) |
| { |
| json_object_object_add(obj, "result", |
| json_object_new_string(result)); |
| } |
| |
| static void add_runtime(struct json_object *obj, double time) |
| { |
| double oldtime; |
| struct json_object *timeobj = get_or_create_json_object(obj, "time"); |
| struct json_object *oldend; |
| |
| json_object_object_add(timeobj, "__type__", |
| json_object_new_string("TimeAttribute")); |
| json_object_object_add(timeobj, "start", |
| json_object_new_double(0.0)); |
| |
| if (!json_object_object_get_ex(timeobj, "end", &oldend)) { |
| json_object_object_add(timeobj, "end", |
| json_object_new_double(time)); |
| return; |
| } |
| |
| /* Add the runtime to the existing runtime. */ |
| oldtime = json_object_get_double(oldend); |
| time += oldtime; |
| json_object_object_add(timeobj, "end", |
| json_object_new_double(time)); |
| } |
| |
| static void set_runtime(struct json_object *obj, double time) |
| { |
| struct json_object *timeobj = get_or_create_json_object(obj, "time"); |
| |
| json_object_object_add(timeobj, "__type__", |
| json_object_new_string("TimeAttribute")); |
| json_object_object_add(timeobj, "start", |
| json_object_new_double(0.0)); |
| json_object_object_add(timeobj, "end", |
| json_object_new_double(time)); |
| } |
| |
| static bool fill_from_output(int fd, const char *binary, const char *key, |
| struct subtests *subtests, |
| struct json_object *tests) |
| { |
| char *buf, *bufend, *nullchr; |
| struct stat statbuf; |
| char piglit_name[256]; |
| char *igt_version = NULL; |
| size_t igt_version_len = 0; |
| struct json_object *current_test = NULL; |
| size_t i; |
| |
| if (fstat(fd, &statbuf)) |
| return false; |
| |
| if (statbuf.st_size != 0) { |
| buf = mmap(NULL, statbuf.st_size, PROT_READ, MAP_SHARED, fd, 0); |
| if (buf == MAP_FAILED) |
| return false; |
| } else { |
| buf = NULL; |
| } |
| |
| /* |
| * Avoid null characters: Just pretend the output stops at the |
| * first such character, if any. |
| */ |
| if ((nullchr = memchr(buf, '\0', statbuf.st_size)) != NULL) { |
| statbuf.st_size = nullchr - buf; |
| } |
| |
| bufend = buf + statbuf.st_size; |
| |
| igt_version = find_line_starting_with(buf, IGT_VERSIONSTRING, bufend); |
| if (igt_version) { |
| char *newline = memchr(igt_version, '\n', bufend - igt_version); |
| igt_version_len = newline - igt_version; |
| } |
| |
| /* TODO: Refactor to helper functions */ |
| if (subtests->size == 0) { |
| /* No subtests */ |
| generate_piglit_name(binary, NULL, piglit_name, sizeof(piglit_name)); |
| current_test = get_or_create_json_object(tests, piglit_name); |
| |
| json_object_object_add(current_test, key, |
| json_object_new_string_len(buf, statbuf.st_size)); |
| if (igt_version) |
| json_object_object_add(current_test, "igt-version", |
| json_object_new_string_len(igt_version, |
| igt_version_len)); |
| |
| return true; |
| } |
| |
| for (i = 0; i < subtests->size; i++) { |
| char *this_sub_begin, *this_sub_result; |
| const char *resulttext; |
| char *beg, *end, *startline; |
| double time; |
| int begin_len; |
| int result_len; |
| |
| generate_piglit_name(binary, subtests->names[i], piglit_name, sizeof(piglit_name)); |
| current_test = get_or_create_json_object(tests, piglit_name); |
| |
| begin_len = asprintf(&this_sub_begin, "%s%s\n", STARTING_SUBTEST, subtests->names[i]); |
| result_len = asprintf(&this_sub_result, "%s%s: ", SUBTEST_RESULT, subtests->names[i]); |
| |
| if (begin_len < 0 || result_len < 0) { |
| fprintf(stderr, "Failure generating strings\n"); |
| return false; |
| } |
| |
| beg = find_line_starting_with(buf, this_sub_begin, bufend); |
| end = find_line_starting_with(buf, this_sub_result, bufend); |
| startline = beg; |
| |
| free(this_sub_begin); |
| free(this_sub_result); |
| |
| if (beg == NULL && end == NULL) { |
| /* No output at all */ |
| beg = bufend; |
| end = bufend; |
| } |
| |
| if (beg == NULL) { |
| /* |
| * Subtest didn't start, probably skipped from |
| * fixture already. Start from the result |
| * line, it gets adjusted below. |
| */ |
| beg = end; |
| } |
| |
| /* Include the output after the previous subtest output */ |
| beg = find_line_after_last(buf, |
| STARTING_SUBTEST, |
| SUBTEST_RESULT, |
| beg); |
| |
| if (end == NULL) { |
| /* Incomplete result. Find the next starting subtest or result. */ |
| end = next_line(startline, bufend); |
| if (end != NULL) { |
| end = find_line_starting_with_either(end, |
| STARTING_SUBTEST, |
| SUBTEST_RESULT, |
| bufend); |
| } |
| if (end == NULL) { |
| end = bufend; |
| } |
| } else { |
| /* |
| * Now pointing to the line where this sub's |
| * result is. We need to include that of |
| * course. |
| */ |
| char *nexttest = next_line(end, bufend); |
| |
| /* Stretch onwards until the next subtest begins or ends */ |
| if (nexttest != NULL) { |
| nexttest = find_line_starting_with_either(nexttest, |
| STARTING_SUBTEST, |
| SUBTEST_RESULT, |
| bufend); |
| } |
| if (nexttest != NULL) { |
| end = nexttest; |
| } else { |
| end = bufend; |
| } |
| } |
| |
| json_object_object_add(current_test, key, |
| json_object_new_string_len(beg, end - beg)); |
| |
| if (igt_version) { |
| json_object_object_add(current_test, "igt-version", |
| json_object_new_string_len(igt_version, |
| igt_version_len)); |
| } |
| |
| if (!json_object_object_get_ex(current_test, "result", NULL)) { |
| parse_subtest_result(subtests->names[i], &resulttext, &time, beg, end); |
| set_result(current_test, resulttext); |
| set_runtime(current_test, time); |
| } |
| } |
| |
| return true; |
| } |
| |
| /* |
| * This regexp controls the kmsg handling. All kernel log records that |
| * have log level of warning or higher convert the result to |
| * dmesg-warn/dmesg-fail unless they match this regexp. |
| * |
| * TODO: Move this to external files, i915-suppressions.txt, |
| * general-suppressions.txt et al. |
| */ |
| |
| #define _ "|" |
| static const char igt_dmesg_whitelist[] = |
| "ACPI: button: The lid device is not compliant to SW_LID" _ |
| "ACPI: .*: Unable to dock!" _ |
| "IRQ [0-9]+: no longer affine to CPU[0-9]+" _ |
| "IRQ fixup: irq [0-9]+ move in progress, old vector [0-9]+" _ |
| /* i915 tests set module options, expected message */ |
| "Setting dangerous option [a-z_]+ - tainting kernel" _ |
| /* Raw printk() call, uses default log level (warn) */ |
| "Suspending console\\(s\\) \\(use no_console_suspend to debug\\)" _ |
| "atkbd serio[0-9]+: Failed to (deactivate|enable) keyboard on isa[0-9]+/serio[0-9]+" _ |
| "cache: parent cpu[0-9]+ should not be sleeping" _ |
| "hpet[0-9]+: lost [0-9]+ rtc interrupts" _ |
| /* i915 selftests terminate normally with ENODEV from the |
| * module load after the testing finishes, which produces this |
| * message. |
| */ |
| "i915: probe of [0-9:.]+ failed with error -25" _ |
| /* swiotbl warns even when asked not to */ |
| "mock: DMA: Out of SW-IOMMU space for [0-9]+ bytes" _ |
| "usb usb[0-9]+: root hub lost power or was reset" |
| ; |
| #undef _ |
| |
| static const char igt_piglit_style_dmesg_blacklist[] = |
| "(\\[drm:|drm_|intel_|i915_)"; |
| |
| static bool init_regex_whitelist(struct settings* settings, GRegex **re) |
| { |
| GError *err = NULL; |
| const char *regex = settings->piglit_style_dmesg ? |
| igt_piglit_style_dmesg_blacklist : |
| igt_dmesg_whitelist; |
| |
| *re = g_regex_new(regex, G_REGEX_OPTIMIZE, 0, &err); |
| if (err) { |
| fprintf(stderr, "Cannot compile dmesg regexp\n"); |
| g_error_free(err); |
| return false; |
| } |
| |
| return true; |
| } |
| |
| static bool parse_dmesg_line(char* line, |
| unsigned *flags, unsigned long long *ts_usec, |
| char *continuation, char **message) |
| { |
| unsigned long long seq; |
| int s; |
| |
| s = sscanf(line, "%u,%llu,%llu,%c;", flags, &seq, ts_usec, continuation); |
| if (s != 4) { |
| /* |
| * Machine readable key/value pairs begin with |
| * a space. We ignore them. |
| */ |
| if (line[0] != ' ') { |
| fprintf(stderr, "Cannot parse kmsg record: %s\n", line); |
| } |
| return false; |
| } |
| |
| *message = strchr(line, ';'); |
| if (!message) { |
| fprintf(stderr, "No ; found in kmsg record, this shouldn't happen\n"); |
| return false; |
| } |
| (*message)++; |
| |
| return true; |
| } |
| |
| static void generate_formatted_dmesg_line(char *message, |
| unsigned flags, |
| unsigned long long ts_usec, |
| char **formatted) |
| { |
| char prefix[512]; |
| size_t messagelen; |
| size_t prefixlen; |
| char *p, *f; |
| |
| snprintf(prefix, sizeof(prefix), |
| "<%u> [%llu.%06llu] ", |
| flags & 0x07, |
| ts_usec / 1000000, |
| ts_usec % 1000000); |
| |
| messagelen = strlen(message); |
| prefixlen = strlen(prefix); |
| |
| /* |
| * Decoding the hex escapes only makes the string shorter, so |
| * we can use the original length |
| */ |
| *formatted = malloc(strlen(prefix) + messagelen + 1); |
| strcpy(*formatted, prefix); |
| |
| f = *formatted + prefixlen; |
| for (p = message; *p; p++, f++) { |
| if (p - message + 4 < messagelen && |
| p[0] == '\\' && p[1] == 'x') { |
| int c = 0; |
| /* newline and tab are not isprint(), but they are isspace() */ |
| if (sscanf(p, "\\x%2x", &c) == 1 && |
| (isprint(c) || isspace(c))) { |
| *f = c; |
| p += 3; |
| continue; |
| } |
| } |
| *f = *p; |
| } |
| *f = '\0'; |
| } |
| |
| static void add_dmesg(struct json_object *obj, |
| const char *dmesg, size_t dmesglen, |
| const char *warnings, size_t warningslen) |
| { |
| json_object_object_add(obj, "dmesg", |
| json_object_new_string_len(dmesg, dmesglen)); |
| |
| if (warnings) { |
| json_object_object_add(obj, "dmesg-warnings", |
| json_object_new_string_len(warnings, warningslen)); |
| } |
| } |
| |
| static void add_empty_dmesgs_where_missing(struct json_object *tests, |
| char *binary, |
| struct subtests *subtests) |
| { |
| struct json_object *current_test; |
| char piglit_name[256]; |
| size_t i; |
| |
| for (i = 0; i < subtests->size; i++) { |
| generate_piglit_name(binary, subtests->names[i], piglit_name, sizeof(piglit_name)); |
| current_test = get_or_create_json_object(tests, piglit_name); |
| if (!json_object_object_get_ex(current_test, "dmesg", NULL)) { |
| add_dmesg(current_test, "", 0, NULL, 0); |
| } |
| } |
| |
| } |
| |
| static bool fill_from_dmesg(int fd, |
| struct settings *settings, |
| char *binary, |
| struct subtests *subtests, |
| struct json_object *tests) |
| { |
| char *line = NULL, *warnings = NULL, *dmesg = NULL; |
| size_t linelen = 0, warningslen = 0, dmesglen = 0; |
| struct json_object *current_test = NULL; |
| FILE *f = fdopen(fd, "r"); |
| char piglit_name[256]; |
| ssize_t read; |
| size_t i; |
| GRegex *re; |
| |
| if (!f) { |
| return false; |
| } |
| |
| if (!init_regex_whitelist(settings, &re)) { |
| fclose(f); |
| return false; |
| } |
| |
| while ((read = getline(&line, &linelen, f)) > 0) { |
| char *formatted; |
| unsigned flags; |
| unsigned long long ts_usec; |
| char continuation; |
| char *message, *subtest; |
| |
| if (!parse_dmesg_line(line, &flags, &ts_usec, &continuation, &message)) |
| continue; |
| |
| generate_formatted_dmesg_line(message, flags, ts_usec, &formatted); |
| |
| if ((subtest = strstr(message, STARTING_SUBTEST_DMESG)) != NULL) { |
| if (current_test != NULL) { |
| /* Done with the previous subtest, file up */ |
| add_dmesg(current_test, dmesg, dmesglen, warnings, warningslen); |
| |
| free(dmesg); |
| free(warnings); |
| dmesg = warnings = NULL; |
| dmesglen = warningslen = 0; |
| } |
| |
| subtest += strlen(STARTING_SUBTEST_DMESG); |
| generate_piglit_name(binary, subtest, piglit_name, sizeof(piglit_name)); |
| current_test = get_or_create_json_object(tests, piglit_name); |
| } |
| |
| if (settings->piglit_style_dmesg) { |
| if ((flags & 0x07) <= settings->dmesg_warn_level && continuation != 'c' && |
| g_regex_match(re, message, 0, NULL)) { |
| append_line(&warnings, &warningslen, formatted); |
| } |
| } else { |
| if ((flags & 0x07) <= settings->dmesg_warn_level && continuation != 'c' && |
| !g_regex_match(re, message, 0, NULL)) { |
| append_line(&warnings, &warningslen, formatted); |
| } |
| } |
| append_line(&dmesg, &dmesglen, formatted); |
| free(formatted); |
| } |
| free(line); |
| |
| if (current_test != NULL) { |
| add_dmesg(current_test, dmesg, dmesglen, warnings, warningslen); |
| } else { |
| /* |
| * Didn't get any subtest messages at all. If there |
| * are subtests, add all of the dmesg gotten to all of |
| * them. |
| */ |
| for (i = 0; i < subtests->size; i++) { |
| generate_piglit_name(binary, subtests->names[i], piglit_name, sizeof(piglit_name)); |
| current_test = get_or_create_json_object(tests, piglit_name); |
| /* |
| * Don't bother with warnings, any subtests |
| * there are would have skip as their result |
| * anyway. |
| */ |
| add_dmesg(current_test, dmesg, dmesglen, NULL, 0); |
| } |
| |
| if (subtests->size == 0) { |
| generate_piglit_name(binary, NULL, piglit_name, sizeof(piglit_name)); |
| current_test = get_or_create_json_object(tests, piglit_name); |
| add_dmesg(current_test, dmesg, dmesglen, warnings, warningslen); |
| } |
| } |
| |
| add_empty_dmesgs_where_missing(tests, binary, subtests); |
| |
| free(dmesg); |
| free(warnings); |
| g_regex_unref(re); |
| fclose(f); |
| return true; |
| } |
| |
| static const char *result_from_exitcode(int exitcode) |
| { |
| switch (exitcode) { |
| case IGT_EXIT_SKIP: |
| return "skip"; |
| case IGT_EXIT_SUCCESS: |
| return "pass"; |
| case IGT_EXIT_INVALID: |
| return "notrun"; |
| case INCOMPLETE_EXITCODE: |
| return "incomplete"; |
| default: |
| return "fail"; |
| } |
| } |
| |
| static void add_subtest(struct subtests *subtests, char *subtest) |
| { |
| size_t len = strlen(subtest); |
| size_t i; |
| |
| if (len == 0) |
| return; |
| |
| if (subtest[len - 1] == '\n') |
| subtest[len - 1] = '\0'; |
| |
| /* Don't add if we already have this subtest */ |
| for (i = 0; i < subtests->size; i++) |
| if (!strcmp(subtest, subtests->names[i])) |
| return; |
| |
| subtests->size++; |
| subtests->names = realloc(subtests->names, sizeof(*subtests->names) * subtests->size); |
| subtests->names[subtests->size - 1] = subtest; |
| } |
| |
| static void free_subtests(struct subtests *subtests) |
| { |
| size_t i; |
| |
| for (i = 0; i < subtests->size; i++) |
| free(subtests->names[i]); |
| free(subtests->names); |
| } |
| |
| static void fill_from_journal(int fd, |
| struct job_list_entry *entry, |
| struct subtests *subtests, |
| struct results *results) |
| { |
| FILE *f = fdopen(fd, "r"); |
| char *line = NULL; |
| size_t linelen = 0; |
| ssize_t read; |
| char exitline[] = "exit:"; |
| char timeoutline[] = "timeout:"; |
| int exitcode = INCOMPLETE_EXITCODE; |
| bool has_timeout = false; |
| struct json_object *tests = results->tests; |
| struct json_object *runtimes = results->runtimes; |
| |
| while ((read = getline(&line, &linelen, f)) > 0) { |
| if (read >= strlen(exitline) && !memcmp(line, exitline, strlen(exitline))) { |
| char *p = strchr(line, '('); |
| char piglit_name[256]; |
| double time = 0.0; |
| struct json_object *obj; |
| |
| exitcode = atoi(line + strlen(exitline)); |
| |
| if (p) |
| time = strtod(p + 1, NULL); |
| |
| generate_piglit_name(entry->binary, NULL, piglit_name, sizeof(piglit_name)); |
| obj = get_or_create_json_object(runtimes, piglit_name); |
| add_runtime(obj, time); |
| |
| /* If no subtests, the test result node also gets the runtime */ |
| if (subtests->size == 0 && entry->subtest_count == 0) { |
| obj = get_or_create_json_object(tests, piglit_name); |
| add_runtime(obj, time); |
| } |
| } else if (read >= strlen(timeoutline) && !memcmp(line, timeoutline, strlen(timeoutline))) { |
| has_timeout = true; |
| |
| if (subtests->size) { |
| /* Assign the timeout to the previously appeared subtest */ |
| char *last_subtest = subtests->names[subtests->size - 1]; |
| char piglit_name[256]; |
| char *p = strchr(line, '('); |
| double time = 0.0; |
| struct json_object *obj; |
| |
| generate_piglit_name(entry->binary, last_subtest, piglit_name, sizeof(piglit_name)); |
| obj = get_or_create_json_object(tests, piglit_name); |
| |
| set_result(obj, "timeout"); |
| |
| if (p) |
| time = strtod(p + 1, NULL); |
| |
| /* Add runtime for the subtest... */ |
| add_runtime(obj, time); |
| |
| /* ... and also for the binary */ |
| generate_piglit_name(entry->binary, NULL, piglit_name, sizeof(piglit_name)); |
| obj = get_or_create_json_object(runtimes, piglit_name); |
| add_runtime(obj, time); |
| } |
| } else { |
| add_subtest(subtests, strdup(line)); |
| } |
| } |
| |
| if (subtests->size == 0) { |
| char *subtestname = NULL; |
| char piglit_name[256]; |
| struct json_object *obj; |
| const char *result = has_timeout ? "timeout" : result_from_exitcode(exitcode); |
| |
| /* |
| * If the test was killed before it printed that it's |
| * entering a subtest, we would incorrectly generate |
| * results as the binary had no subtests. If we know |
| * otherwise, do otherwise. |
| */ |
| if (entry->subtest_count > 0) { |
| subtestname = entry->subtests[0]; |
| add_subtest(subtests, strdup(subtestname)); |
| } |
| |
| generate_piglit_name(entry->binary, subtestname, piglit_name, sizeof(piglit_name)); |
| obj = get_or_create_json_object(tests, piglit_name); |
| set_result(obj, result); |
| } |
| |
| free(line); |
| fclose(f); |
| } |
| |
| static void override_result_single(struct json_object *obj) |
| { |
| const char *errtext = "", *result = ""; |
| struct json_object *textobj; |
| bool dmesgwarns = false; |
| |
| if (json_object_object_get_ex(obj, "err", &textobj)) |
| errtext = json_object_get_string(textobj); |
| if (json_object_object_get_ex(obj, "result", &textobj)) |
| result = json_object_get_string(textobj); |
| if (json_object_object_get_ex(obj, "dmesg-warnings", &textobj)) |
| dmesgwarns = true; |
| |
| if (!strcmp(result, "pass") && |
| count_lines(errtext, errtext + strlen(errtext)) > 2) { |
| set_result(obj, "warn"); |
| result = "warn"; |
| } |
| |
| if (dmesgwarns) { |
| if (!strcmp(result, "pass") || !strcmp(result, "warn")) { |
| set_result(obj, "dmesg-warn"); |
| } else if (!strcmp(result, "fail")) { |
| set_result(obj, "dmesg-fail"); |
| } |
| } |
| } |
| |
| static void override_results(char *binary, |
| struct subtests *subtests, |
| struct json_object *tests) |
| { |
| struct json_object *obj; |
| char piglit_name[256]; |
| size_t i; |
| |
| if (subtests->size == 0) { |
| generate_piglit_name(binary, NULL, piglit_name, sizeof(piglit_name)); |
| obj = get_or_create_json_object(tests, piglit_name); |
| override_result_single(obj); |
| return; |
| } |
| |
| for (i = 0; i < subtests->size; i++) { |
| generate_piglit_name(binary, subtests->names[i], piglit_name, sizeof(piglit_name)); |
| obj = get_or_create_json_object(tests, piglit_name); |
| override_result_single(obj); |
| } |
| } |
| |
| static struct json_object *get_totals_object(struct json_object *totals, |
| const char *key) |
| { |
| struct json_object *obj = NULL; |
| |
| if (json_object_object_get_ex(totals, key, &obj)) |
| return obj; |
| |
| obj = json_object_new_object(); |
| json_object_object_add(totals, key, obj); |
| |
| json_object_object_add(obj, "crash", json_object_new_int(0)); |
| json_object_object_add(obj, "pass", json_object_new_int(0)); |
| json_object_object_add(obj, "dmesg-fail", json_object_new_int(0)); |
| json_object_object_add(obj, "dmesg-warn", json_object_new_int(0)); |
| json_object_object_add(obj, "skip", json_object_new_int(0)); |
| json_object_object_add(obj, "incomplete", json_object_new_int(0)); |
| json_object_object_add(obj, "timeout", json_object_new_int(0)); |
| json_object_object_add(obj, "notrun", json_object_new_int(0)); |
| json_object_object_add(obj, "fail", json_object_new_int(0)); |
| json_object_object_add(obj, "warn", json_object_new_int(0)); |
| |
| return obj; |
| } |
| |
| static void add_result_to_totals(struct json_object *totals, |
| const char *result) |
| { |
| json_object *numobj = NULL; |
| int old; |
| |
| if (!json_object_object_get_ex(totals, result, &numobj)) { |
| fprintf(stderr, "Warning: Totals object without count for %s\n", result); |
| return; |
| } |
| |
| old = json_object_get_int(numobj); |
| json_object_object_add(totals, result, json_object_new_int(old + 1)); |
| } |
| |
| static void add_to_totals(const char *binary, |
| struct subtests *subtests, |
| struct results *results) |
| { |
| struct json_object *test, *resultobj, *emptystrtotal, *roottotal, *binarytotal; |
| char piglit_name[256]; |
| const char *result; |
| size_t i; |
| |
| generate_piglit_name(binary, NULL, piglit_name, sizeof(piglit_name)); |
| emptystrtotal = get_totals_object(results->totals, ""); |
| roottotal = get_totals_object(results->totals, "root"); |
| binarytotal = get_totals_object(results->totals, piglit_name); |
| |
| if (subtests->size == 0) { |
| test = get_or_create_json_object(results->tests, piglit_name); |
| if (!json_object_object_get_ex(test, "result", &resultobj)) { |
| fprintf(stderr, "Warning: No results set for %s\n", piglit_name); |
| return; |
| } |
| result = json_object_get_string(resultobj); |
| add_result_to_totals(emptystrtotal, result); |
| add_result_to_totals(roottotal, result); |
| add_result_to_totals(binarytotal, result); |
| return; |
| } |
| |
| for (i = 0; i < subtests->size; i++) { |
| generate_piglit_name(binary, subtests->names[i], piglit_name, sizeof(piglit_name)); |
| test = get_or_create_json_object(results->tests, piglit_name); |
| if (!json_object_object_get_ex(test, "result", &resultobj)) { |
| fprintf(stderr, "Warning: No results set for %s\n", piglit_name); |
| return; |
| } |
| result = json_object_get_string(resultobj); |
| add_result_to_totals(emptystrtotal, result); |
| add_result_to_totals(roottotal, result); |
| add_result_to_totals(binarytotal, result); |
| } |
| } |
| |
| static bool parse_test_directory(int dirfd, |
| struct job_list_entry *entry, |
| struct settings *settings, |
| struct results *results) |
| { |
| int fds[_F_LAST]; |
| struct subtests subtests = {}; |
| bool status = true; |
| |
| if (!open_output_files(dirfd, fds, false)) { |
| fprintf(stderr, "Error opening output files\n"); |
| return false; |
| } |
| |
| /* |
| * fill_from_journal fills the subtests struct and adds |
| * timeout results where applicable. |
| */ |
| fill_from_journal(fds[_F_JOURNAL], entry, &subtests, results); |
| |
| if (!fill_from_output(fds[_F_OUT], entry->binary, "out", &subtests, results->tests) || |
| !fill_from_output(fds[_F_ERR], entry->binary, "err", &subtests, results->tests) || |
| !fill_from_dmesg(fds[_F_DMESG], settings, entry->binary, &subtests, results->tests)) { |
| fprintf(stderr, "Error parsing output files\n"); |
| status = false; |
| goto parse_output_end; |
| } |
| |
| override_results(entry->binary, &subtests, results->tests); |
| add_to_totals(entry->binary, &subtests, results); |
| |
| parse_output_end: |
| close_outputs(fds); |
| free_subtests(&subtests); |
| |
| return status; |
| } |
| |
| static void try_add_notrun_results(const struct job_list_entry *entry, |
| const struct settings *settings, |
| struct results *results) |
| { |
| struct subtests subtests = {}; |
| struct json_object *current_test; |
| size_t i; |
| |
| if (entry->subtest_count == 0) { |
| char piglit_name[256]; |
| |
| /* We cannot distinguish no-subtests from run-all-subtests in multiple-mode */ |
| if (settings->multiple_mode) |
| return; |
| generate_piglit_name(entry->binary, NULL, piglit_name, sizeof(piglit_name)); |
| current_test = get_or_create_json_object(results->tests, piglit_name); |
| json_object_object_add(current_test, "out", json_object_new_string("")); |
| json_object_object_add(current_test, "err", json_object_new_string("")); |
| json_object_object_add(current_test, "dmesg", json_object_new_string("")); |
| json_object_object_add(current_test, "result", json_object_new_string("notrun")); |
| } |
| |
| for (i = 0; i < entry->subtest_count; i++) { |
| char piglit_name[256]; |
| |
| generate_piglit_name(entry->binary, entry->subtests[i], piglit_name, sizeof(piglit_name)); |
| current_test = get_or_create_json_object(results->tests, piglit_name); |
| json_object_object_add(current_test, "out", json_object_new_string("")); |
| json_object_object_add(current_test, "err", json_object_new_string("")); |
| json_object_object_add(current_test, "dmesg", json_object_new_string("")); |
| json_object_object_add(current_test, "result", json_object_new_string("notrun")); |
| add_subtest(&subtests, strdup(entry->subtests[i])); |
| } |
| |
| add_to_totals(entry->binary, &subtests, results); |
| free_subtests(&subtests); |
| } |
| |
| static void create_result_root_nodes(struct json_object *root, |
| struct results *results) |
| { |
| results->tests = json_object_new_object(); |
| json_object_object_add(root, "tests", results->tests); |
| results->totals = json_object_new_object(); |
| json_object_object_add(root, "totals", results->totals); |
| results->runtimes = json_object_new_object(); |
| json_object_object_add(root, "runtimes", results->runtimes); |
| } |
| |
| struct json_object *generate_results_json(int dirfd) |
| { |
| struct settings settings; |
| struct job_list job_list; |
| struct json_object *obj, *elapsed; |
| struct results results; |
| int testdirfd, fd; |
| size_t i; |
| |
| init_settings(&settings); |
| init_job_list(&job_list); |
| |
| if (!read_settings_from_dir(&settings, dirfd)) { |
| fprintf(stderr, "resultgen: Cannot parse settings\n"); |
| return NULL; |
| } |
| |
| if (!read_job_list(&job_list, dirfd)) { |
| fprintf(stderr, "resultgen: Cannot parse job list\n"); |
| return NULL; |
| } |
| |
| obj = json_object_new_object(); |
| json_object_object_add(obj, "__type__", json_object_new_string("TestrunResult")); |
| json_object_object_add(obj, "results_version", json_object_new_int(10)); |
| json_object_object_add(obj, "name", |
| settings.name ? |
| json_object_new_string(settings.name) : |
| json_object_new_string("")); |
| |
| if ((fd = openat(dirfd, "uname.txt", O_RDONLY)) >= 0) { |
| char buf[128]; |
| ssize_t r = read(fd, buf, sizeof(buf)); |
| |
| if (r > 0 && buf[r - 1] == '\n') |
| r--; |
| |
| json_object_object_add(obj, "uname", |
| json_object_new_string_len(buf, r)); |
| close(fd); |
| } |
| |
| elapsed = json_object_new_object(); |
| json_object_object_add(elapsed, "__type__", json_object_new_string("TimeAttribute")); |
| if ((fd = openat(dirfd, "starttime.txt", O_RDONLY)) >= 0) { |
| char buf[128] = {}; |
| read(fd, buf, sizeof(buf)); |
| json_object_object_add(elapsed, "start", json_object_new_double(atof(buf))); |
| close(fd); |
| } |
| if ((fd = openat(dirfd, "endtime.txt", O_RDONLY)) >= 0) { |
| char buf[128] = {}; |
| read(fd, buf, sizeof(buf)); |
| json_object_object_add(elapsed, "end", json_object_new_double(atof(buf))); |
| close(fd); |
| } |
| json_object_object_add(obj, "time_elapsed", elapsed); |
| |
| create_result_root_nodes(obj, &results); |
| |
| /* |
| * Result fields that won't be added: |
| * |
| * - glxinfo |
| * - wglinfo |
| * - clinfo |
| * |
| * Result fields that are TODO: |
| * |
| * - lspci |
| * - options |
| */ |
| |
| for (i = 0; i < job_list.size; i++) { |
| char name[16]; |
| |
| snprintf(name, 16, "%zd", i); |
| if ((testdirfd = openat(dirfd, name, O_DIRECTORY | O_RDONLY)) < 0) { |
| try_add_notrun_results(&job_list.entries[i], &settings, &results); |
| continue; |
| } |
| |
| if (!parse_test_directory(testdirfd, &job_list.entries[i], &settings, &results)) { |
| close(testdirfd); |
| return NULL; |
| } |
| close(testdirfd); |
| } |
| |
| if ((fd = openat(dirfd, "aborted.txt", O_RDONLY)) >= 0) { |
| char buf[4096]; |
| char piglit_name[] = "igt@runner@aborted"; |
| struct subtests abortsub = {}; |
| struct json_object *aborttest = get_or_create_json_object(results.tests, piglit_name); |
| ssize_t s; |
| |
| add_subtest(&abortsub, strdup("aborted")); |
| |
| s = read(fd, buf, sizeof(buf)); |
| |
| json_object_object_add(aborttest, "out", |
| json_object_new_string_len(buf, s)); |
| json_object_object_add(aborttest, "err", |
| json_object_new_string("")); |
| json_object_object_add(aborttest, "dmesg", |
| json_object_new_string("")); |
| json_object_object_add(aborttest, "result", |
| json_object_new_string("fail")); |
| |
| add_to_totals("runner", &abortsub, &results); |
| |
| free_subtests(&abortsub); |
| } |
| |
| free_settings(&settings); |
| free_job_list(&job_list); |
| |
| return obj; |
| } |
| |
| bool generate_results(int dirfd) |
| { |
| struct json_object *obj = generate_results_json(dirfd); |
| const char *json_string; |
| int resultsfd; |
| |
| if (obj == NULL) |
| return false; |
| |
| /* TODO: settings.overwrite */ |
| if ((resultsfd = openat(dirfd, "results.json", O_WRONLY | O_CREAT | O_TRUNC, 0666)) < 0) { |
| fprintf(stderr, "resultgen: Cannot create results file\n"); |
| return false; |
| } |
| |
| json_string = json_object_to_json_string_ext(obj, JSON_C_TO_STRING_PRETTY); |
| write(resultsfd, json_string, strlen(json_string)); |
| close(resultsfd); |
| return true; |
| } |
| |
| bool generate_results_path(char *resultspath) |
| { |
| int dirfd = open(resultspath, O_DIRECTORY | O_RDONLY); |
| |
| if (dirfd < 0) |
| return false; |
| |
| return generate_results(dirfd); |
| } |