src/main/java/com/android/vts/job/VtsSuiteTestJobServlet.java - platform/test/vti/dashboard - Git at Google

 /*
  * Copyright (c) 2018 Google Inc. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you
  * may not use this file except in compliance with the License. You may
  * obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  * implied. See the License for the specific language governing
  * permissions and limitations under the License.
  */

 package com.android.vts.job;

 import com.android.vts.entity.TestSuiteFileEntity;
 import com.android.vts.entity.TestSuiteResultEntity;
 import com.android.vts.proto.TestSuiteResultMessageProto;
 import com.android.vts.util.GcsHelper;
 import com.android.vts.util.TaskQueueHelper;
 import com.android.vts.util.TimeUtil;
 import com.google.appengine.api.memcache.ErrorHandlers;
 import com.google.appengine.api.memcache.MemcacheService;
 import com.google.appengine.api.memcache.MemcacheServiceFactory;
 import com.google.appengine.api.taskqueue.Queue;
 import com.google.appengine.api.taskqueue.QueueFactory;
 import com.google.appengine.api.taskqueue.TaskOptions;
 import com.google.cloud.storage.Blob;
 import com.google.cloud.storage.Bucket;
 import com.google.cloud.storage.Storage;
 import com.googlecode.objectify.Key;
 import javax.servlet.ServletConfig;
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.FileSystems;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;

 import static com.googlecode.objectify.ObjectifyService.ofy;

 /** Suite Test result file processing job. */
 public class VtsSuiteTestJobServlet extends BaseJobServlet {

     private static final String SUITE_TEST_URL = "/cron/test_suite_report_gcs_monitor";

     private static final String SERVICE_NAME = "VTS Dashboard";

     private final Logger logger = Logger.getLogger(this.getClass().getName());

     /** Google Cloud Storage project's key file to access the storage */
     private static String GCS_KEY_FILE;
     /** Google Cloud Storage project's default bucket name for vtslab log files */
     private static String GCS_BUCKET_NAME;
     /** Google Cloud Storage project's default directory name for suite test result files */
     private static String GCS_SUITE_TEST_FOLDER_NAME;

     public static final String QUEUE = "suiteTestQueue";

     /**
      * This is the key file to access vtslab-gcs project. It will allow the dashboard to have a full
      * control of the bucket.
      */
     private InputStream keyFileInputStream;

     /** This is the instance of java google storage library */
     private Storage storage;

     /** This is the instance of App Engine memcache service java library */
     private MemcacheService syncCache = MemcacheServiceFactory.getMemcacheService();

     @Override
     public void init(ServletConfig servletConfig) throws ServletException {
         super.init(servletConfig);

         GCS_KEY_FILE = systemConfigProp.getProperty("gcs.keyFile");
         GCS_BUCKET_NAME = systemConfigProp.getProperty("gcs.bucketName");
         GCS_SUITE_TEST_FOLDER_NAME = systemConfigProp.getProperty("gcs.suiteTestFolderName");

         this.keyFileInputStream =
                 this.getClass().getClassLoader().getResourceAsStream("keys/" + GCS_KEY_FILE);

         Optional<Storage> optionalStorage = GcsHelper.getStorage(this.keyFileInputStream);
         if (optionalStorage.isPresent()) {
             this.storage = optionalStorage.get();
         } else {
             logger.log(Level.SEVERE, "Error on getting storage instance!");
             throw new ServletException("Creating storage instance exception!");
         }
         syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
     }

     @Override
     public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException {

         List<String> dateStringList = new ArrayList<>();

         long currentMicroSecond = TimeUnit.MILLISECONDS.toMicros(System.currentTimeMillis());

         ZonedDateTime checkZonedDateTime = TimeUtil.getZonedDateTime(currentMicroSecond);
         String checkDateString =
                 DateTimeFormatter.ofPattern(TimeUtil.DATE_FORMAT)
                         .format(checkZonedDateTime.minusMinutes(5));
         String todayDateString = TimeUtil.getDateString(currentMicroSecond);
         if (!checkDateString.equals(todayDateString)) {
             dateStringList.add(checkDateString);
             logger.log(Level.INFO, "Yesterday is added to the process queue and processed!");
         }
         dateStringList.add(todayDateString);

         for (String dateString : dateStringList) {
             String[] dateArray = dateString.split("-");
             if (dateArray.length == 3) {

                 Queue queue = QueueFactory.getQueue(QUEUE);

                 List<TaskOptions> tasks = new ArrayList<>();

                 String fileSeparator = FileSystems.getDefault().getSeparator();

                 String year = dateArray[0];
                 String month = dateArray[1];
                 String day = dateArray[2];

                 List<String> pathList = Arrays.asList(GCS_SUITE_TEST_FOLDER_NAME, year, month, day);
                 Path pathInfo = Paths.get(String.join(fileSeparator, pathList));

                 List<TestSuiteFileEntity> testSuiteFileEntityList =
                         ofy().load()
                                 .type(TestSuiteFileEntity.class)
                                 .filter("year", Integer.parseInt(year))
                                 .filter("month", Integer.parseInt(month))
                                 .filter("day", Integer.parseInt(day))
                                 .list();

                 List<String> filePathList =
                         testSuiteFileEntityList
                                 .stream()
                                 .map(testSuiteFile -> testSuiteFile.getFilePath())
                                 .collect(Collectors.toList());

                 Bucket vtsReportBucket = this.storage.get(GCS_BUCKET_NAME);

                 Storage.BlobListOption[] listOptions =
                         new Storage.BlobListOption[] {
                             Storage.BlobListOption.prefix(pathInfo.toString() + fileSeparator)
                         };

                 Iterable<Blob> blobIterable = vtsReportBucket.list(listOptions).iterateAll();
                 Iterator<Blob> blobIterator = blobIterable.iterator();
                 while (blobIterator.hasNext()) {
                     Blob blob = blobIterator.next();
                     if (blob.isDirectory()) {
                         logger.log(Level.INFO, blob.getName() + " directory will be skipped!");
                     } else {
                         if (filePathList.contains(blob.getName())) {
                             logger.log(Level.INFO, "filePathList contain => " + blob.getName());
                         } else if (blob.getName().endsWith(fileSeparator)) {
                             logger.log(Level.INFO, blob.getName() + " endswith slash!");
                         } else {
                             TaskOptions task =
                                     TaskOptions.Builder.withUrl(SUITE_TEST_URL)
                                             .param("filePath", blob.getName())
                                             .method(TaskOptions.Method.POST);
                             tasks.add(task);
                         }
                     }
                 }
                 TaskQueueHelper.addToQueue(queue, tasks);
             } else {
                 throw new IllegalArgumentException(
                         todayDateString + " date string not in correct format");
             }
         }
     }

     @Override
     public void doPost(HttpServletRequest request, HttpServletResponse response)
             throws IOException {

         String filePath = request.getParameter("filePath");
         if (Objects.isNull(filePath)) {
             logger.log(Level.WARNING, "filePath parameter is null!");
         } else {
             logger.log(Level.INFO, "filePath parameter => " + filePath);

             Key<TestSuiteFileEntity> testSuiteFileEntityKey =
                     Key.create(TestSuiteFileEntity.class, filePath);
             TestSuiteFileEntity testSuiteFileEntity =
                     ofy().load()
                             .type(TestSuiteFileEntity.class)
                             .filterKey(testSuiteFileEntityKey)
                             .first()
                             .now();

             if (Objects.isNull(testSuiteFileEntity)) {
                 Blob blobFile = (Blob) this.syncCache.get(filePath);
                 if (Objects.isNull(blobFile)) {
                     Bucket vtsReportBucket = this.storage.get(GCS_BUCKET_NAME);
                     blobFile = vtsReportBucket.get(filePath);
                     this.syncCache.put(filePath, blobFile);
                 }

                 if (blobFile.exists()) {
                     TestSuiteFileEntity newTestSuiteFileEntity = new TestSuiteFileEntity(filePath);
                     try {
                         TestSuiteResultMessageProto.TestSuiteResultMessage testSuiteResultMessage =
                                 TestSuiteResultMessageProto.TestSuiteResultMessage.parseFrom(
                                         blobFile.getContent());

                         TestSuiteResultEntity testSuiteResultEntity =
                                 new TestSuiteResultEntity(
                                         testSuiteFileEntityKey,
                                         testSuiteResultMessage.getStartTime(),
                                         testSuiteResultMessage.getEndTime(),
                                         testSuiteResultMessage.getTestType(),
                                         testSuiteResultMessage.getBootSuccess(),
                                         testSuiteResultMessage.getResultPath(),
                                         testSuiteResultMessage.getInfraLogPath(),
                                         testSuiteResultMessage.getHostName(),
                                         testSuiteResultMessage.getSuitePlan(),
                                         testSuiteResultMessage.getSuiteVersion(),
                                         testSuiteResultMessage.getSuiteName(),
                                         testSuiteResultMessage.getSuiteBuildNumber(),
                                         testSuiteResultMessage.getModulesDone(),
                                         testSuiteResultMessage.getModulesTotal(),
                                         testSuiteResultMessage.getBranch(),
                                         testSuiteResultMessage.getTarget(),
                                         testSuiteResultMessage.getBuildId(),
                                         testSuiteResultMessage.getBuildSystemFingerprint(),
                                         testSuiteResultMessage.getBuildVendorFingerprint(),
                                         testSuiteResultMessage.getPassedTestCaseCount(),
                                         testSuiteResultMessage.getFailedTestCaseCount());

                         testSuiteResultEntity.save(newTestSuiteFileEntity);
                     } catch (IOException e) {
                         ofy().delete().type(TestSuiteFileEntity.class).id(filePath).now();
                         response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
                         logger.log(Level.WARNING, "Invalid proto: " + e.getLocalizedMessage());
                     }
                 }
             } else {
                 logger.log(Level.INFO, "suite test found in datastore!");
             }
         }
     }
 }
	/*
	* Copyright (c) 2018 Google Inc. All Rights Reserved.
	*
	* Licensed under the Apache License, Version 2.0 (the "License"); you
	* may not use this file except in compliance with the License. You may
	* obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
	* implied. See the License for the specific language governing
	* permissions and limitations under the License.
	*/

	package com.android.vts.job;

	import com.android.vts.entity.TestSuiteFileEntity;
	import com.android.vts.entity.TestSuiteResultEntity;
	import com.android.vts.proto.TestSuiteResultMessageProto;
	import com.android.vts.util.GcsHelper;
	import com.android.vts.util.TaskQueueHelper;
	import com.android.vts.util.TimeUtil;
	import com.google.appengine.api.memcache.ErrorHandlers;
	import com.google.appengine.api.memcache.MemcacheService;
	import com.google.appengine.api.memcache.MemcacheServiceFactory;
	import com.google.appengine.api.taskqueue.Queue;
	import com.google.appengine.api.taskqueue.QueueFactory;
	import com.google.appengine.api.taskqueue.TaskOptions;
	import com.google.cloud.storage.Blob;
	import com.google.cloud.storage.Bucket;
	import com.google.cloud.storage.Storage;
	import com.googlecode.objectify.Key;
	import javax.servlet.ServletConfig;
	import javax.servlet.ServletException;
	import javax.servlet.http.HttpServletRequest;
	import javax.servlet.http.HttpServletResponse;
	import java.io.IOException;
	import java.io.InputStream;
	import java.nio.file.FileSystems;
	import java.nio.file.Path;
	import java.nio.file.Paths;
	import java.time.ZonedDateTime;
	import java.time.format.DateTimeFormatter;
	import java.util.ArrayList;
	import java.util.Arrays;
	import java.util.Iterator;
	import java.util.List;
	import java.util.Objects;
	import java.util.Optional;
	import java.util.concurrent.TimeUnit;
	import java.util.logging.Level;
	import java.util.logging.Logger;
	import java.util.stream.Collectors;

	import static com.googlecode.objectify.ObjectifyService.ofy;

	/** Suite Test result file processing job. */
	public class VtsSuiteTestJobServlet extends BaseJobServlet {

	private static final String SUITE_TEST_URL = "/cron/test_suite_report_gcs_monitor";

	private static final String SERVICE_NAME = "VTS Dashboard";

	private final Logger logger = Logger.getLogger(this.getClass().getName());

	/** Google Cloud Storage project's key file to access the storage */
	private static String GCS_KEY_FILE;
	/** Google Cloud Storage project's default bucket name for vtslab log files */
	private static String GCS_BUCKET_NAME;
	/** Google Cloud Storage project's default directory name for suite test result files */
	private static String GCS_SUITE_TEST_FOLDER_NAME;

	public static final String QUEUE = "suiteTestQueue";

	/**
	* This is the key file to access vtslab-gcs project. It will allow the dashboard to have a full
	* control of the bucket.
	*/
	private InputStream keyFileInputStream;

	/** This is the instance of java google storage library */
	private Storage storage;

	/** This is the instance of App Engine memcache service java library */
	private MemcacheService syncCache = MemcacheServiceFactory.getMemcacheService();

	@Override
	public void init(ServletConfig servletConfig) throws ServletException {
	super.init(servletConfig);

	GCS_KEY_FILE = systemConfigProp.getProperty("gcs.keyFile");
	GCS_BUCKET_NAME = systemConfigProp.getProperty("gcs.bucketName");
	GCS_SUITE_TEST_FOLDER_NAME = systemConfigProp.getProperty("gcs.suiteTestFolderName");

	this.keyFileInputStream =
	this.getClass().getClassLoader().getResourceAsStream("keys/" + GCS_KEY_FILE);

	Optional<Storage> optionalStorage = GcsHelper.getStorage(this.keyFileInputStream);
	if (optionalStorage.isPresent()) {
	this.storage = optionalStorage.get();
	} else {
	logger.log(Level.SEVERE, "Error on getting storage instance!");
	throw new ServletException("Creating storage instance exception!");
	}
	syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
	}

	@Override
	public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException {

	List<String> dateStringList = new ArrayList<>();

	long currentMicroSecond = TimeUnit.MILLISECONDS.toMicros(System.currentTimeMillis());

	ZonedDateTime checkZonedDateTime = TimeUtil.getZonedDateTime(currentMicroSecond);
	String checkDateString =
	DateTimeFormatter.ofPattern(TimeUtil.DATE_FORMAT)
	.format(checkZonedDateTime.minusMinutes(5));
	String todayDateString = TimeUtil.getDateString(currentMicroSecond);
	if (!checkDateString.equals(todayDateString)) {
	dateStringList.add(checkDateString);
	logger.log(Level.INFO, "Yesterday is added to the process queue and processed!");
	}
	dateStringList.add(todayDateString);

	for (String dateString : dateStringList) {
	String[] dateArray = dateString.split("-");
	if (dateArray.length == 3) {

	Queue queue = QueueFactory.getQueue(QUEUE);

	List<TaskOptions> tasks = new ArrayList<>();

	String fileSeparator = FileSystems.getDefault().getSeparator();

	String year = dateArray[0];
	String month = dateArray[1];
	String day = dateArray[2];

	List<String> pathList = Arrays.asList(GCS_SUITE_TEST_FOLDER_NAME, year, month, day);
	Path pathInfo = Paths.get(String.join(fileSeparator, pathList));

	List<TestSuiteFileEntity> testSuiteFileEntityList =
	ofy().load()
	.type(TestSuiteFileEntity.class)
	.filter("year", Integer.parseInt(year))
	.filter("month", Integer.parseInt(month))
	.filter("day", Integer.parseInt(day))
	.list();

	List<String> filePathList =
	testSuiteFileEntityList
	.stream()
	.map(testSuiteFile -> testSuiteFile.getFilePath())
	.collect(Collectors.toList());

	Bucket vtsReportBucket = this.storage.get(GCS_BUCKET_NAME);

	Storage.BlobListOption[] listOptions =
	new Storage.BlobListOption[] {
	Storage.BlobListOption.prefix(pathInfo.toString() + fileSeparator)
	};

	Iterable<Blob> blobIterable = vtsReportBucket.list(listOptions).iterateAll();
	Iterator<Blob> blobIterator = blobIterable.iterator();
	while (blobIterator.hasNext()) {
	Blob blob = blobIterator.next();
	if (blob.isDirectory()) {
	logger.log(Level.INFO, blob.getName() + " directory will be skipped!");
	} else {
	if (filePathList.contains(blob.getName())) {
	logger.log(Level.INFO, "filePathList contain => " + blob.getName());
	} else if (blob.getName().endsWith(fileSeparator)) {
	logger.log(Level.INFO, blob.getName() + " endswith slash!");
	} else {
	TaskOptions task =
	TaskOptions.Builder.withUrl(SUITE_TEST_URL)
	.param("filePath", blob.getName())
	.method(TaskOptions.Method.POST);
	tasks.add(task);
	}
	}
	}
	TaskQueueHelper.addToQueue(queue, tasks);
	} else {
	throw new IllegalArgumentException(
	todayDateString + " date string not in correct format");
	}
	}
	}

	@Override
	public void doPost(HttpServletRequest request, HttpServletResponse response)
	throws IOException {

	String filePath = request.getParameter("filePath");
	if (Objects.isNull(filePath)) {
	logger.log(Level.WARNING, "filePath parameter is null!");
	} else {
	logger.log(Level.INFO, "filePath parameter => " + filePath);

	Key<TestSuiteFileEntity> testSuiteFileEntityKey =
	Key.create(TestSuiteFileEntity.class, filePath);
	TestSuiteFileEntity testSuiteFileEntity =
	ofy().load()
	.type(TestSuiteFileEntity.class)
	.filterKey(testSuiteFileEntityKey)
	.first()
	.now();

	if (Objects.isNull(testSuiteFileEntity)) {
	Blob blobFile = (Blob) this.syncCache.get(filePath);
	if (Objects.isNull(blobFile)) {
	Bucket vtsReportBucket = this.storage.get(GCS_BUCKET_NAME);
	blobFile = vtsReportBucket.get(filePath);
	this.syncCache.put(filePath, blobFile);
	}

	if (blobFile.exists()) {
	TestSuiteFileEntity newTestSuiteFileEntity = new TestSuiteFileEntity(filePath);
	try {
	TestSuiteResultMessageProto.TestSuiteResultMessage testSuiteResultMessage =
	TestSuiteResultMessageProto.TestSuiteResultMessage.parseFrom(
	blobFile.getContent());

	TestSuiteResultEntity testSuiteResultEntity =
	new TestSuiteResultEntity(
	testSuiteFileEntityKey,
	testSuiteResultMessage.getStartTime(),
	testSuiteResultMessage.getEndTime(),
	testSuiteResultMessage.getTestType(),
	testSuiteResultMessage.getBootSuccess(),
	testSuiteResultMessage.getResultPath(),
	testSuiteResultMessage.getInfraLogPath(),
	testSuiteResultMessage.getHostName(),
	testSuiteResultMessage.getSuitePlan(),
	testSuiteResultMessage.getSuiteVersion(),
	testSuiteResultMessage.getSuiteName(),
	testSuiteResultMessage.getSuiteBuildNumber(),
	testSuiteResultMessage.getModulesDone(),
	testSuiteResultMessage.getModulesTotal(),
	testSuiteResultMessage.getBranch(),
	testSuiteResultMessage.getTarget(),
	testSuiteResultMessage.getBuildId(),
	testSuiteResultMessage.getBuildSystemFingerprint(),
	testSuiteResultMessage.getBuildVendorFingerprint(),
	testSuiteResultMessage.getPassedTestCaseCount(),
	testSuiteResultMessage.getFailedTestCaseCount());

	testSuiteResultEntity.save(newTestSuiteFileEntity);
	} catch (IOException e) {
	ofy().delete().type(TestSuiteFileEntity.class).id(filePath).now();
	response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
	logger.log(Level.WARNING, "Invalid proto: " + e.getLocalizedMessage());
	}
	}
	} else {
	logger.log(Level.INFO, "suite test found in datastore!");
	}
	}
	}
	}