diff --git a/build.gradle b/build.gradle index 284d37592..43f7f5a17 100644 --- a/build.gradle +++ b/build.gradle @@ -38,10 +38,11 @@ subprojects { } allprojects { + apply plugin: 'java' + sourceCompatibility = '1.8' + targetCompatibility = '1.8' ext { - sourceCompatibility = '1.8' - targetCompatibility = '1.8' jettyVer = '9.4.12.v20180830' @@ -58,13 +59,19 @@ allprojects { slf4jVer = '1.7.25' gsonVer = '2.8.2' + + junitVer = '4.12' + mockitoVer = '2.22.0' } repositories { //uncomment for RC testing - maven { - url "https://repository.apache.org/content/repositories/orgapacheignite-1463/" - } + // maven { + // url "https://repository.apache.org/content/repositories/orgapacheignite-1463/" + // } + + mavenCentral() + mavenLocal() } } diff --git a/ignite-tc-helper-web/build.gradle b/ignite-tc-helper-web/build.gradle index 9f5c512f2..e4fa0116a 100644 --- a/ignite-tc-helper-web/build.gradle +++ b/ignite-tc-helper-web/build.gradle @@ -18,14 +18,6 @@ apply plugin: 'java' apply plugin: 'war' -sourceCompatibility = '1.8' -targetCompatibility = '1.8' - -repositories { - mavenCentral() - mavenLocal() -} - // https://www.apache.org/legal/resolved.html#category-a dependencies { compile (project(":tcbot-common")); @@ -72,12 +64,8 @@ dependencies { } processResources { - - // Minify json resources from(sourceSets.test.resources.srcDirs) { include "**/*.json" - - // Minify every file here } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/TcHelperDb.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/TcHelperDb.java index 14642ccdc..e706b3346 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/TcHelperDb.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/TcHelperDb.java @@ -111,15 +111,6 @@ public static CacheConfiguration getCacheV2TxConfig(String name) { } - @NotNull - public static CacheConfiguration getCache8PartsConfig(String name) { - CacheConfiguration ccfg = new CacheConfiguration<>(name); - - ccfg.setAffinity(new RendezvousAffinityFunction(false, 8)); - - return ccfg; - } - public static class LocalOnlyTcpDiscoveryIpFinder implements TcpDiscoveryIpFinder { /** Port. */ private int port; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/IgniteTcBotModule.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/IgniteTcBotModule.java index bcbc2e7ed..dcceff13d 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/IgniteTcBotModule.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/IgniteTcBotModule.java @@ -28,9 +28,9 @@ import org.apache.ignite.Ignite; import org.apache.ignite.ci.db.Ignite1Init; import org.apache.ignite.ci.di.cache.GuavaCachedModule; -import org.apache.ignite.ci.di.scheduler.SchedulerModule; -import org.apache.ignite.ci.github.ignited.GitHubIgnitedModule; -import org.apache.ignite.ci.jira.ignited.JiraIgnitedModule; +import org.apache.ignite.tcbot.persistence.scheduler.SchedulerModule; +import org.apache.ignite.githubignited.GitHubIgnitedModule; +import org.apache.ignite.jiraignited.JiraIgnitedModule; import org.apache.ignite.ci.observer.BuildObserver; import org.apache.ignite.ci.observer.ObserverTask; import org.apache.ignite.ci.tcbot.TcBotBusinessServicesModule; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/ChangeUi.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/ChangeUi.java index 11496d926..5b433b572 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/ChangeUi.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/ChangeUi.java @@ -17,6 +17,8 @@ package org.apache.ignite.ci.issue; +import com.google.common.base.MoreObjects; + public class ChangeUi { public final String username; public final String webUrl; @@ -41,4 +43,12 @@ public String toSlackMarkup() { public String toPlainText() { return username + " " + webUrl; } + + /** {@inheritDoc} */ + @Override public String toString() { + return MoreObjects.toStringHelper(this) + .add("username", username) + .add("webUrl", webUrl) + .toString(); + } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/Issue.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/Issue.java index 05236a008..2e99c52d5 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/Issue.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/Issue.java @@ -20,24 +20,27 @@ import com.google.common.base.MoreObjects; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Objects; import java.util.Set; +import java.util.Map; import java.util.TreeSet; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import org.apache.ignite.tcbot.persistence.Persisted; import org.apache.ignite.tcbot.common.util.TimeUtil; -import org.jetbrains.annotations.Nullable; /** - * + * Issue used both for saving into DB and in UI (in issue history). */ -@SuppressWarnings("WeakerAccess") +@SuppressWarnings({"WeakerAccess", "PublicField"}) @Persisted public class Issue { /** Type code. Null of older versions of issue */ @Nullable - private String type; + public String type; /** Display type. for issue. Kept for backward compatibilty with older records without type code. */ private String displayType; @@ -57,7 +60,7 @@ public class Issue { @Nullable public String displayName; - /** Build start timestamp. */ + /** Build start timestamp. Builds which is older that 10 days not notified. */ @Nullable public Long buildStartTs; /** Detected timestamp. */ @@ -66,11 +69,21 @@ public class Issue { /** Set of build tags detected. */ public Set buildTags = new TreeSet<>(); - public Issue(IssueKey issueKey, IssueType type) { + /** Statistics of subscribers for this issue. Filled accordignly recent update. */ + public Map stat = new HashMap<>(); + + /** Notification failed: Map from address to exception text */ + @Nullable public Map notificationFailed = new HashMap<>(); + + public int notificationRetry = 0; + + public Issue(IssueKey issueKey, IssueType type, + @Nullable Long buildStartTs) { this.issueKey = issueKey; this.detectedTs = System.currentTimeMillis(); this.type = type.code(); this.displayType = type.displayName(); + this.buildStartTs = buildStartTs; } public void addChange(String username, String webUrl) { @@ -230,4 +243,32 @@ public String toPlainText(boolean includeChangesInfo) { public String type() { return type; } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + Issue issue = (Issue)o; + return notificationRetry == issue.notificationRetry && + Objects.equals(type, issue.type) && + Objects.equals(displayType, issue.displayType) && + Objects.equals(trackedBranchName, issue.trackedBranchName) && + Objects.equals(issueKey, issue.issueKey) && + Objects.equals(changes, issue.changes) && + Objects.equals(addressNotified, issue.addressNotified) && + Objects.equals(webUrl, issue.webUrl) && + Objects.equals(displayName, issue.displayName) && + Objects.equals(buildStartTs, issue.buildStartTs) && + Objects.equals(detectedTs, issue.detectedTs) && + Objects.equals(buildTags, issue.buildTags) && + Objects.equals(stat, issue.stat) && + Objects.equals(notificationFailed, issue.notificationFailed); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return Objects.hash(type, displayType, trackedBranchName, issueKey, changes, addressNotified, webUrl, displayName, buildStartTs, detectedTs, buildTags, stat, notificationFailed, notificationRetry); + } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/IssuesStorage.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/IssuesStorage.java index d13d9d401..a18294d77 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/IssuesStorage.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/IssuesStorage.java @@ -17,8 +17,10 @@ package org.apache.ignite.ci.issue; +import java.util.HashMap; import java.util.stream.Stream; import java.util.stream.StreamSupport; +import javax.annotation.Nullable; import javax.cache.Cache; import javax.inject.Inject; import javax.inject.Provider; @@ -52,17 +54,60 @@ public static IgniteCache botDetectedIssuesCache(Ignite ignite) } /** {@inheritDoc} */ - @Override public boolean setNotified(IssueKey issueKey, String to) { + @Override public boolean getIsNewAndSetNotified(IssueKey issueKey, String to, + @Nullable Exception e) { Issue issue = cache().get(issueKey); if (issue == null) return false; - boolean add = issue.addressNotified.add(to); + boolean update; + if (e == null) { + if (issue.notificationRetry >= 2 && issue.notificationFailed.containsKey(to)) + return false; // no more tries; - if (add) + update = issue.addressNotified.add(to); + + if (update && issue.notificationFailed != null) + issue.notificationFailed.remove(to); + } + else { + if (issue.notificationRetry < 2) + issue.addressNotified.remove(to); + + issue.notificationRetry++; + + if (issue.notificationFailed == null) + issue.notificationFailed = new HashMap<>(); + + issue.notificationFailed.put(to, e.getClass().getSimpleName() + ": " + e.getMessage()); + + update = true; + } + + if (update) cache().put(issueKey, issue); - return add; + return update; + } + + @Override + public void saveIssueSubscribersStat(IssueKey issueKey, int cntSrvAllowed, int cntSubscribed, + int cntTagsFilterPassed) { + Issue issue = cache().get(issueKey); + if (issue == null) + return; + + int hashCode = issue.hashCode(); + + if (issue.stat == null) + issue.stat = new HashMap<>(); + + issue.stat.put("cntSrvAllowed", cntSrvAllowed); + issue.stat.put("cntSubscribed", cntSubscribed); + issue.stat.put("cntTagsFilterPassed", cntTagsFilterPassed); + + if (issue.hashCode() != hashCode) + cache().put(issueKey, issue); // protect from odd writes } /** {@inheritDoc} */ diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/mail/EmailSender.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/mail/EmailSender.java index 3f8f07593..895d59668 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/mail/EmailSender.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/mail/EmailSender.java @@ -35,8 +35,8 @@ * Class for sending email with configured credentials. */ public class EmailSender { - public static boolean sendEmail(String to, String subject, String html, String plainText, - NotificationsConfig notifications) { + public static void sendEmail(String to, String subject, String html, String plainText, + NotificationsConfig notifications) throws MessagingException { String user = notifications.emailUsernameMandatory(); @@ -58,42 +58,34 @@ public static boolean sendEmail(String to, String subject, String html, String p } }); - try { - // Create a default MimeMessage object. - MimeMessage msg = new MimeMessage(ses); + // Create a default MimeMessage object. + MimeMessage msg = new MimeMessage(ses); - // Set From: header field of the header. - msg.setFrom(new InternetAddress(from)); + // Set From: header field of the header. + msg.setFrom(new InternetAddress(from)); - // Set To: header field of the header. - msg.addRecipient(Message.RecipientType.TO, new InternetAddress(to)); + // Set To: header field of the header. + msg.addRecipient(Message.RecipientType.TO, new InternetAddress(to)); - // Set Subject: header field - msg.setSubject(subject); + // Set Subject: header field + msg.setSubject(subject); - final MimeBodyPart textPart = new MimeBodyPart(); - textPart.setContent(plainText, "text/plain"); - // HTML version - final MimeBodyPart htmlPart = new MimeBodyPart(); - htmlPart.setContent(html, "text/html"); + final MimeBodyPart textPart = new MimeBodyPart(); + textPart.setContent(plainText, "text/plain"); + // HTML version + final MimeBodyPart htmlPart = new MimeBodyPart(); + htmlPart.setContent(html, "text/html"); - // Create the Multipart. Add BodyParts to it. - final Multipart mp = new MimeMultipart("alternative"); - mp.addBodyPart(textPart); - mp.addBodyPart(htmlPart); - // Set Multipart as the message's content - msg.setContent(mp); + // Create the Multipart. Add BodyParts to it. + final Multipart mp = new MimeMultipart("alternative"); + mp.addBodyPart(textPart); + mp.addBodyPart(htmlPart); + // Set Multipart as the message's content + msg.setContent(mp); - // Send message - Transport.send(msg); + // Send message + Transport.send(msg); - System.out.println("Sent message successfully to [" + to + "]..."); - - return true; - } - catch (MessagingException mex) { - mex.printStackTrace(); - } - return false; + System.out.println("Sent message successfully to [" + to + "]..."); } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/mail/SlackSender.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/mail/SlackSender.java index d15d337e6..fa8e43b63 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/mail/SlackSender.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/mail/SlackSender.java @@ -34,7 +34,7 @@ * */ public class SlackSender { - public static boolean sendMessage(String addr, String msg, + public static void sendMessage(String addr, String msg, NotificationsConfig cfg) throws IOException { String authTok = cfg.slackAuthToken(); Preconditions.checkState(!isNullOrEmpty(authTok), "notifications:\"{}\" property should be filled in branches.json"); @@ -49,11 +49,8 @@ public static boolean sendMessage(String addr, String msg, SlackChannel slackCh = ses.findChannelByName(ch); - if (slackCh == null) { - System.err.println("Failed to find channel [" + addr + "]: Notification not send [" + msg + "]"); - - return false; - } + if (slackCh == null) + throw new RuntimeException("Failed to find channel [" + addr + "]: Notification not send [" + msg + "]"); SlackMessageHandle handle = ses.sendMessage(slackCh, msg); @@ -62,22 +59,16 @@ public static boolean sendMessage(String addr, String msg, else { SlackUser user = ses.findUserByUserName(addr); //make sure bot is a member of the user. - if (user == null) { - System.err.println("Failed to find user [" + addr + "]: Notification not send [" + msg + "]"); - - return false; - } + if (user == null) + throw new RuntimeException("Failed to find user [" + addr + "]: Notification not send [" + msg + "]"); SlackMessageHandle handle = ses.sendMessageToUser(user, msg, null); System.out.println("Message to user " + addr + " " + msg + "; acked: " + handle.isAcked()); - } } finally { ses.disconnect(); } - - return true; } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/ClientTmpHelper.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/ClientTmpHelper.java index e799ed281..75774d2d5 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/ClientTmpHelper.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/ClientTmpHelper.java @@ -20,15 +20,20 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.ci.db.TcHelperDb; -import org.apache.ignite.ci.github.ignited.IGitHubConnIgnited; +import org.apache.ignite.githubignited.IGitHubConnIgnited; import org.apache.ignite.ci.issue.Issue; import org.apache.ignite.ci.issue.IssuesStorage; -import org.apache.ignite.ci.jira.ignited.JiraTicketDao; +import org.apache.ignite.jiraignited.JiraTicketDao; /** * Utility class for local connection to TC helper DB (server) and any manipulations with data needed. */ public class ClientTmpHelper { + public static void main(String[] args) { + //select some main option here. + mainDropIssHist(args); + } + /** * @param args Args. */ @@ -47,7 +52,7 @@ public static void mainConsistCheck(String[] args) { /** * @param args Args. */ - public static void main0(String[] args) { + public static void mainDropIssHist(String[] args) { Ignite ignite = TcHelperDb.startClient(); IgniteCache cache = ignite.cache(IssuesStorage.BOT_DETECTED_ISSUES); @@ -56,7 +61,10 @@ public static void main0(String[] args) { issue -> { Object key = issue.getKey(); Issue val = (Issue)issue.getValue(); - // val.addressNotified.clear(); + + if(val.issueKey.testOrBuildName.contains( + "GridCachePartitionEvictionDuringReadThroughSelfTest.testPartitionRent")) + val.addressNotified.clear(); cache.put(key, val); } @@ -74,9 +82,6 @@ public static void mainDeletePRs(String[] args) { ignite.close(); } - public static void main(String[] args) { - mainDeleteTickets(args); - } public static void mainDeleteTickets(String[] args) { Ignite ignite = TcHelperDb.startClient(); diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java index de23a2680..3d5a9e91a 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java @@ -19,33 +19,40 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; -import java.io.BufferedWriter; import java.io.IOException; import java.io.UncheckedIOException; +import java.time.Duration; +import java.util.Iterator; +import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; import javax.cache.Cache; import javax.xml.bind.JAXBException; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.Ignition; +import org.apache.ignite.ci.issue.Issue; +import org.apache.ignite.ci.issue.IssueKey; +import org.apache.ignite.ci.issue.IssueType; +import org.apache.ignite.ci.issue.IssuesStorage; import org.apache.ignite.ci.tcbot.user.UserAndSessionsStorage; -import org.apache.ignite.tcservice.model.hist.BuildRef; -import org.apache.ignite.tcservice.model.result.Build; import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted; -import org.apache.ignite.tcignited.ITeamcityIgnited; -import org.apache.ignite.tcbot.persistence.IgniteStringCompactor; -import org.apache.ignite.tcignited.buildref.BuildRefDao; import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; -import org.apache.ignite.tcignited.build.FatBuildDao; -import org.apache.ignite.tcignited.history.RunHistCompactedDao; import org.apache.ignite.ci.user.TcHelperUser; -import org.apache.ignite.tcservice.util.XmlUtil; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.logger.slf4j.Slf4jLogger; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.tcbot.persistence.IgniteStringCompactor; +import org.apache.ignite.tcignited.ITeamcityIgnited; +import org.apache.ignite.tcignited.build.FatBuildDao; +import org.apache.ignite.tcignited.buildref.BuildRefDao; +import org.apache.ignite.tcignited.history.RunHistCompactedDao; +import org.apache.ignite.tcservice.model.hist.BuildRef; +import org.apache.ignite.tcservice.model.result.Build; +import org.apache.ignite.tcservice.util.XmlUtil; import org.jetbrains.annotations.NotNull; import static org.apache.ignite.tcignited.history.RunHistCompactedDao.BUILD_START_TIME_CACHE_NAME; @@ -62,9 +69,145 @@ public class RemoteClientTmpHelper { * @param args Args. */ public static void main(String[] args) { + // mainDropInvalidIssues(args); System.err.println("Please insert option of main"); } + public static void mainDropInvalidIssues(String[] args) { + try (Ignite ignite = tcbotServerConnectedClient()) { + IgniteCache bst = ignite.cache(IssuesStorage.BOT_DETECTED_ISSUES); + Iterator> iter = bst.iterator(); + try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dumpsDir(), + "Issues_dropped.txt")))) { + while (iter.hasNext()) { + Cache.Entry next = iter.next(); + + boolean rmv = false; + Issue val = next.getValue(); + if (val.type == null) + rmv = true; + + //don't touch it + if (Objects.equals(val.type, IssueType.newContributedTestFailure.code())) + continue; + + long ageDays = -1; + if (val != null && val.buildStartTs == null) { + if (val.detectedTs == null) + rmv = true; + else + ageDays = Duration.ofMillis(System.currentTimeMillis() - val.detectedTs).toDays(); + + + rmv = true; + } + + if(rmv) { + bst.remove(next.getKey()); + + String str = "Removing issue " + next.getKey() + " " + val.type + " detected " + + ageDays + " days ago\n"; + writer.write(str); + System.err.println(str); + } + } + + writer.flush(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + + dumpDictionary(ignite); + } + } + + + public static void mainDumpIssues(String[] args) { + try (Ignite ignite = tcbotServerConnectedClient()) { + IgniteCache bst = ignite.cache(IssuesStorage.BOT_DETECTED_ISSUES); + Iterator> iter = bst.iterator(); + try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dumpsDir(), + "Issues.txt")))) { + while (iter.hasNext()) { + Cache.Entry next = iter.next(); + + Issue val = next.getValue(); + long ageDays = -1; + if (val != null && val.buildStartTs != null) + ageDays = Duration.ofMillis(System.currentTimeMillis() - val.buildStartTs).toDays(); + + writer.write(next.getKey() + " " + val + " " + + ageDays + "\n"); + } + + writer.flush(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + + dumpDictionary(ignite); + } + } + + + public static void mainDumpFatBuildStartTime(String[] args) { + try (Ignite ignite = tcbotServerConnectedClient()) { + IgniteCache bst = ignite.cache(FatBuildDao.TEAMCITY_FAT_BUILD_CACHE_NAME); + Iterator> iterator = bst.iterator(); + try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dumpsDir(), + "fatBuildStartTime.txt")))) { + while (iterator.hasNext()) { + Cache.Entry next = iterator.next(); + + FatBuildCompacted val = next.getValue(); + long ageDays = -1; + long startDateTs = -2; + + if (val != null) { + startDateTs = val.getStartDateTs(); + ageDays = Duration.ofMillis(System.currentTimeMillis() - startDateTs).toDays(); + } + + writer.write(next.getKey() + " " + startDateTs + " " + + ageDays + "\n"); + } + + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + + + public static void mainDumpBuildStartTime(String[] args) { + try (Ignite ignite = tcbotServerConnectedClient()) { + IgniteCache bst = ignite.cache(BUILD_START_TIME_CACHE_NAME); + Iterator> iterator = bst.iterator(); + try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dumpsDir(), + "BuildStartTime.txt")))) { + while (iterator.hasNext()) { + Cache.Entry next = iterator.next(); + + Long val = next.getValue(); + long ageDays = -1; + if(val!=null) + ageDays = Duration.ofMillis(System.currentTimeMillis() - val).toDays(); + + writer.write(next.getKey() + " " + val + " " + + ageDays +"\n"); + } + + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + + /** * @param args Args. */ @@ -195,18 +338,12 @@ private static void setupDisco(IgniteConfiguration cfg) { cfg.setDiscoverySpi(spi); } - public static void mainExport(String[] args) throws IOException { + public static void mainExport(String[] args) { final Ignite ignite = tcbotServerConnectedClient(); - if (dumpDict) { - IgniteCache strings = ignite.cache(IgniteStringCompactor.STRINGS_CACHE); - try (FileWriter writer = new FileWriter("Dictionary.txt")) { - for (Cache.Entry next1 : strings) { - writer.write(next1.getValue().toString() - + "\n"); - } - } - } + if (dumpDict) + dumpDictionary(ignite); + if (false) { IgniteCache cache1 = ignite.cache(FatBuildDao.TEAMCITY_FAT_BUILD_CACHE_NAME); @@ -223,6 +360,21 @@ public static void mainExport(String[] args) throws IOException { } } + public static void dumpDictionary(Ignite ignite) { + IgniteCache strings = ignite.cache(IgniteStringCompactor.STRINGS_CACHE); + try { + try (FileWriter writer = new FileWriter(new File(dumpsDir(), "Dictionary.txt"))) { + for (Cache.Entry next1 : strings) { + writer.write(next1.getValue().toString() + + "\n"); + } + } + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + public static Ignite tcbotServerConnectedClient() { final IgniteConfiguration cfg = new IgniteConfiguration(); diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/TcBotBusinessServicesModule.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/TcBotBusinessServicesModule.java index 2134a3f4c..f0d636f00 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/TcBotBusinessServicesModule.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/TcBotBusinessServicesModule.java @@ -19,6 +19,7 @@ import com.google.inject.AbstractModule; import com.google.inject.internal.SingletonScope; import org.apache.ignite.ci.issue.IssuesStorage; +import org.apache.ignite.tcbot.engine.buildtime.BuildTimeService; import org.apache.ignite.tcbot.engine.chain.BuildChainProcessor; import org.apache.ignite.tcbot.engine.conf.ITcBotConfig; import org.apache.ignite.ci.tcbot.conf.LocalFilesBasedConfig; @@ -42,5 +43,8 @@ public class TcBotBusinessServicesModule extends AbstractModule { bind(MasterTrendsService.class).in(new SingletonScope()); bind(ITcBotBgAuth.class).to(TcBotBgAuthImpl.class).in(new SingletonScope()); bind(BuildChainProcessor.class).in(new SingletonScope()); + + //todo move to bot engine module + bind(BuildTimeService.class).in(new SingletonScope()); } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/conf/JiraServerConfig.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/conf/JiraServerConfig.java index 0de7cd514..a18b9eabe 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/conf/JiraServerConfig.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/conf/JiraServerConfig.java @@ -19,7 +19,7 @@ import com.google.common.base.Strings; import java.util.Properties; import org.apache.ignite.ci.HelperConfig; -import org.apache.ignite.ci.jira.pure.Ticket; +import org.apache.ignite.jiraservice.Ticket; import org.apache.ignite.tcbot.common.conf.IJiraServerConfig; import org.apache.ignite.tcbot.common.conf.PasswordEncoder; import org.jetbrains.annotations.Nullable; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IIssuesStorage.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IIssuesStorage.java index 46a023e42..af7a992d6 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IIssuesStorage.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IIssuesStorage.java @@ -18,6 +18,7 @@ package org.apache.ignite.ci.tcbot.issue; import java.util.stream.Stream; +import javax.annotation.Nullable; import org.apache.ignite.ci.issue.Issue; import org.apache.ignite.ci.issue.IssueKey; @@ -38,7 +39,10 @@ public interface IIssuesStorage { * Checks and saves address was notified (NotThreadSafe) * @param key issue key. * @param addr Address to register as notified. + * @param e Exception. Null means notification was successfull. Non null means notification failed * @return update successful. This address was not notified before. */ - public boolean setNotified(IssueKey key, String addr); + public boolean getIsNewAndSetNotified(IssueKey key, String addr, @Nullable Exception e); + + public void saveIssueSubscribersStat(IssueKey key, int cntSrvAllowed, int cntSubscribed, int cntTagsFilterPassed); } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java index e3800a764..ad0c6304c 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java @@ -19,28 +19,44 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; -import org.apache.ignite.ci.issue.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import javax.annotation.Nonnull; +import javax.inject.Inject; +import javax.inject.Provider; +import org.apache.ignite.ci.issue.Issue; +import org.apache.ignite.ci.issue.IssueKey; +import org.apache.ignite.ci.issue.IssueType; import org.apache.ignite.ci.jobs.CheckQueueJob; import org.apache.ignite.ci.mail.EmailSender; import org.apache.ignite.ci.mail.SlackSender; -import org.apache.ignite.tcbot.engine.issue.EventTemplate; -import org.apache.ignite.tcbot.engine.issue.EventTemplates; -import org.apache.ignite.tcbot.engine.tracked.TrackedBranchChainsProcessor; import org.apache.ignite.ci.tcbot.user.IUserStorage; import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted; import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; -import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData; import org.apache.ignite.ci.user.ITcBotUserCreds; import org.apache.ignite.ci.user.TcHelperUser; -import org.apache.ignite.tcbot.engine.ui.DsChainUi; -import org.apache.ignite.tcbot.engine.ui.DsSuiteUi; -import org.apache.ignite.tcbot.engine.ui.DsTestFailureUi; -import org.apache.ignite.tcbot.engine.ui.DsSummaryUi; +import org.apache.ignite.tcbot.common.TcBotConst; import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; import org.apache.ignite.tcbot.common.interceptor.MonitoredTask; import org.apache.ignite.tcbot.engine.conf.INotificationChannel; import org.apache.ignite.tcbot.engine.conf.ITcBotConfig; import org.apache.ignite.tcbot.engine.conf.NotificationsConfig; +import org.apache.ignite.tcbot.engine.issue.EventTemplate; +import org.apache.ignite.tcbot.engine.issue.EventTemplates; +import org.apache.ignite.tcbot.engine.tracked.TrackedBranchChainsProcessor; +import org.apache.ignite.tcbot.engine.ui.DsChainUi; +import org.apache.ignite.tcbot.engine.ui.DsSuiteUi; +import org.apache.ignite.tcbot.engine.ui.DsSummaryUi; +import org.apache.ignite.tcbot.engine.ui.DsTestFailureUi; import org.apache.ignite.tcbot.persistence.IStringCompactor; import org.apache.ignite.tcignited.ITeamcityIgnited; import org.apache.ignite.tcignited.ITeamcityIgnitedProvider; @@ -50,17 +66,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nonnull; -import javax.inject.Inject; -import javax.inject.Provider; -import java.io.IOException; -import java.util.*; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch; /** @@ -130,7 +135,7 @@ private void sendNewNotifications() { @SuppressWarnings({"WeakerAccess", "UnusedReturnValue"}) @AutoProfiling @MonitoredTask(name = "Send Notifications") - protected String sendNewNotificationsEx() throws IOException { + protected String sendNewNotificationsEx() { List channels = new ArrayList<>(); userStorage.allUsers() @@ -143,6 +148,11 @@ protected String sendNewNotificationsEx() throws IOException { Map toBeSent = new HashMap<>(); AtomicInteger issuesChecked = new AtomicInteger(); + AtomicInteger filteredFresh = new AtomicInteger(); + AtomicInteger filteredBuildTs = new AtomicInteger(); + AtomicInteger filteredNotDisabled = new AtomicInteger(); + AtomicInteger hasSubscriptions = new AtomicInteger(); + AtomicInteger neverSentBefore = new AtomicInteger(); issuesStorage.allIssues() .peek(issue -> issuesChecked.incrementAndGet()) @@ -150,40 +160,63 @@ protected String sendNewNotificationsEx() throws IOException { long detected = issue.detectedTs == null ? 0 : issue.detectedTs; long issueAgeMs = System.currentTimeMillis() - detected; - return issueAgeMs <= TimeUnit.HOURS.toMillis(2); + //here boundary can be not an absolute, but some ts when particular notification channel config was changed + // alternatively boundary may depend to issue notification histroy + + boolean neverNotified = issue.addressNotified == null || issue.addressNotified.isEmpty(); + // if issue had a prior notification, limit age by 2 hours to avoid new addresses spamming. + // otherwise check last day issues if it is notifiable + long bound = TimeUnit.HOURS.toMillis(neverNotified + ? TcBotConst.NOTIFY_MAX_AGE_SINCE_DETECT_HOURS + : TcBotConst.NOTIFY_MAX_AGE_SINCE_DETECT_FOR_NOTIFIED_ISSUE_HOURS ); + + return issueAgeMs <= bound; }) + .peek(issue -> filteredFresh.incrementAndGet()) .filter(issue -> { + if (issue.buildStartTs == null) + return true; // exception due to bug in issue detection; field was not filled + long buildStartTs = issue.buildStartTs == null ? 0 : issue.buildStartTs; long buildAgeMs = System.currentTimeMillis() - buildStartTs; - long maxBuildAgeToNotify = TimeUnit.DAYS.toMillis(InvocationData.MAX_DAYS) / 2; + long maxBuildAgeToNotify = TimeUnit.DAYS.toMillis(TcBotConst.NOTIFY_MAX_AGE_SINCE_START_DAYS) / 2; return buildAgeMs <= maxBuildAgeToNotify; }) + .peek(issue -> filteredBuildTs.incrementAndGet()) .filter(issue -> { return cfg.getTrackedBranches() .get(issue.trackedBranchName) .filter(tb -> !tb.disableIssueTypes().contains(issue.type())) .isPresent(); }) + .peek(issue -> filteredNotDisabled.incrementAndGet()) .forEach(issue -> { List addrs = new ArrayList<>(); final String srvCode = issue.issueKey().server; + AtomicInteger ctnSrvAllowed = new AtomicInteger(); + AtomicInteger cntSubscibed = new AtomicInteger(); + AtomicInteger cntTagsFilterPassed = new AtomicInteger(); + channels.stream() .filter(ch -> ch.isServerAllowed(srvCode)) + .peek(ch -> ctnSrvAllowed.incrementAndGet()) .filter(ch -> ch.isSubscribedToBranch(issue.trackedBranchName)) + .peek(ch -> cntSubscibed.incrementAndGet()) .filter(ch -> { if (ch.hasTagFilter()) return issue.buildTags().stream().anyMatch(ch::isSubscribedToTag); return true; }) + .peek(ch -> cntTagsFilterPassed.incrementAndGet()) .forEach(channel -> { String email = channel.email(); String slack = channel.slack(); logger.info("User/channel " + channel + " is candidate for notification " + email - + " , " + slack + "for " + issue); + + " , " + slack + " for " + issue); if (!Strings.isNullOrEmpty(email)) addrs.add(email); @@ -192,8 +225,15 @@ protected String sendNewNotificationsEx() throws IOException { addrs.add(SLACK + slack); }); + if(!addrs.isEmpty()) + hasSubscriptions.incrementAndGet(); + + boolean nonNotifedChFound = false; + for (String nextAddr : addrs) { - if (issuesStorage.setNotified(issue.issueKey, nextAddr)) { + if (issuesStorage.getIsNewAndSetNotified(issue.issueKey, nextAddr, null)) { + nonNotifedChFound = true; + toBeSent.computeIfAbsent(nextAddr, addr -> { Notification notification = new Notification(); notification.ts = System.currentTimeMillis(); @@ -202,39 +242,70 @@ protected String sendNewNotificationsEx() throws IOException { }).addIssue(issue); } } + + if (!nonNotifedChFound) { + issuesStorage.saveIssueSubscribersStat(issue.issueKey, + ctnSrvAllowed.get(), + cntSubscibed.get(), + cntTagsFilterPassed.get()); + } + else + neverSentBefore.incrementAndGet(); }); + String stat = issuesChecked.get() + " issues checked, " + + filteredFresh.get() + " detected recenty, " + + filteredBuildTs.get() + " for fresh builds, " + + filteredNotDisabled.get() + " not disabled, " + + hasSubscriptions.get() + " has subscriber, " + + neverSentBefore.get() + " non sent before"; + if (toBeSent.isEmpty()) - return "Noting to notify, " + issuesChecked + " issues checked"; + return "Noting to notify, " + stat; NotificationsConfig notifications = cfg.notifications(); - StringBuilder res = new StringBuilder(); - Collection values = toBeSent.values(); - for (Notification next : values) { - if (next.addr.startsWith(SLACK)) { - String slackUser = next.addr.substring(SLACK.length()); + Map sndStat = new HashMap<>(); + + for (Notification next : toBeSent.values()) { + String addr = next.addr; + + try { + if (addr.startsWith(SLACK)) { + String slackUser = addr.substring(SLACK.length()); + + List messages = next.toSlackMarkup(); - List messages = next.toSlackMarkup(); + for (String msg : messages) { + SlackSender.sendMessage(slackUser, msg, notifications); - for (String msg : messages) { - final boolean snd = SlackSender.sendMessage(slackUser, msg, notifications); + sndStat.computeIfAbsent(addr, k -> new AtomicInteger()).incrementAndGet(); + } + } + else { + String builds = next.buildIdToIssue.keySet().toString(); + String subj = "[MTCGA]: " + next.countIssues() + " new failures in builds " + builds + " needs to be handled"; + + EmailSender.sendEmail(addr, subj, next.toHtml(), next.toPlainText(), notifications); - res.append("Send ").append(slackUser).append(": ").append(snd); - if (!snd) - break; + sndStat.computeIfAbsent(addr, k -> new AtomicInteger()).incrementAndGet(); } } - else { - String builds = next.buildIdToIssue.keySet().toString(); - String subj = "[MTCGA]: " + next.countIssues() + " new failures in builds " + builds + " needs to be handled"; + catch (Exception e) { + e.printStackTrace(); + logger.warn("Unable to notify address [" + addr + "] about build failures", e); + + next.allIssues().forEach(issue -> { + IssueKey key = issue.issueKey(); + // rollback successfull notification + issuesStorage.getIsNewAndSetNotified(key, addr, e); + }); - EmailSender.sendEmail(next.addr, subj, next.toHtml(), next.toPlainText(), notifications); - res.append("Send ").append(next.addr).append(" subject: ").append(subj); + stat += " ;" + e.getClass().getSimpleName() + ": " + e.getMessage(); } } - return res + ", " + issuesChecked.get() + "issues checked"; + return "Send " + sndStat.toString() + "; Statistics: " + stat; } /** @@ -261,13 +332,14 @@ protected String registerNewIssues(DsSummaryUi res, ITcBotUserCreds creds) { final String trackedBranch = res.getTrackedBranch(); + String suiteId = suiteCurrentStatus.suiteId; for (DsTestFailureUi testFailure : suiteCurrentStatus.testFailures) { - if (registerTestFailIssues(tcIgnited, srvCode, normalizeBranch, testFailure, trackedBranch, + if (registerTestFailIssues(tcIgnited, srvCode, suiteId, normalizeBranch, testFailure, trackedBranch, suiteCurrentStatus.tags)) newIssues++; } - if (registerSuiteFailIssues(tcIgnited, srvCode, normalizeBranch, suiteCurrentStatus, trackedBranch)) + if (registerSuiteFailIssues(tcIgnited, srvCode, suiteId, normalizeBranch, suiteCurrentStatus, trackedBranch)) newIssues++; } } @@ -286,13 +358,15 @@ protected String registerNewIssues(DsSummaryUi res, ITcBotUserCreds creds) { */ private boolean registerSuiteFailIssues(ITeamcityIgnited tcIgnited, String srvCode, + String suiteId, String normalizeBranch, DsSuiteUi suiteFailure, String trackedBranch) { - String suiteId = suiteFailure.suiteId; + Integer btId = compactor.getStringIdIfPresent(suiteId); + Integer brNormId = compactor.getStringIdIfPresent(normalizeBranch); - IRunHistory runStat = tcIgnited.getSuiteRunHist(suiteId, normalizeBranch); + IRunHistory runStat = tcIgnited.getSuiteRunHist(btId, brNormId).self(); if (runStat == null) return false; @@ -334,11 +408,10 @@ private boolean registerSuiteFailIssues(ITeamcityIgnited tcIgnited, @NotNull private Issue createIssueForSuite(ITeamcityIgnited tcIgnited, DsSuiteUi suiteFailure, String trackedBranch, IssueKey issueKey, IssueType issType) { - Issue issue = new Issue(issueKey, issType); + Issue issue = new Issue(issueKey, issType, tcIgnited.getBuildStartTs(issueKey.buildId)); issue.trackedBranchName = trackedBranch; issue.displayName = suiteFailure.name; issue.webUrl = suiteFailure.webToHist; - issue.buildStartTs = tcIgnited.getBuildStartTs(issueKey.buildId); issue.buildTags.addAll(suiteFailure.tags); @@ -361,13 +434,17 @@ private void locateChanges(ITeamcityIgnited teamcity, int buildId, Issue issue) private boolean registerTestFailIssues(ITeamcityIgnited tcIgnited, String srvCode, + String suiteId, String normalizeBranch, DsTestFailureUi testFailure, String trackedBranch, @Nonnull Set suiteTags) { String name = testFailure.name; + int tname = compactor.getStringId(name); + Integer btId = compactor.getStringIdIfPresent(suiteId); + Integer brNormId = compactor.getStringIdIfPresent(normalizeBranch); - IRunHistory runStat = tcIgnited.getTestRunHist(name, normalizeBranch); + IRunHistory runStat = tcIgnited.getTestRunHist(tname, btId, brNormId); if (runStat == null) return false; @@ -412,7 +489,7 @@ private boolean registerTestFailIssues(ITeamcityIgnited tcIgnited, if (issuesStorage.containsIssueKey(issueKey)) return false; //duplicate - Issue issue = new Issue(issueKey, type); + Issue issue = new Issue(issueKey, type, tcIgnited.getBuildStartTs(issueKey.buildId)); issue.trackedBranchName = trackedBranch; issue.displayName = testFailure.testName; issue.webUrl = testFailure.webUrl; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/Notification.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/Notification.java index 21ebb72d8..87fba013e 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/Notification.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/Notification.java @@ -17,6 +17,7 @@ package org.apache.ignite.ci.tcbot.issue; +import java.util.stream.Stream; import org.apache.ignite.ci.issue.Issue; import org.apache.ignite.ci.web.model.Version; import org.apache.ignite.tcbot.common.util.TimeUtil; @@ -193,4 +194,7 @@ private String toSlackMarkup(List issues) { return sb.toString(); } + public Stream allIssues() { + return this.buildIdToIssue.values().stream().flatMap(Collection::stream); + } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/TcBotTriggerAndSignOffService.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/TcBotTriggerAndSignOffService.java index cd17eaa9d..da034a01f 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/TcBotTriggerAndSignOffService.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/TcBotTriggerAndSignOffService.java @@ -37,19 +37,21 @@ import javax.annotation.Nonnull; import javax.inject.Inject; import javax.ws.rs.QueryParam; + +import org.apache.ignite.tcbot.engine.pr.BranchTicketMatcher; import org.apache.ignite.tcservice.ITeamcity; import org.apache.ignite.ci.github.GitHubBranch; import org.apache.ignite.ci.github.GitHubUser; import org.apache.ignite.ci.github.PullRequest; -import org.apache.ignite.ci.github.ignited.IGitHubConnIgnited; -import org.apache.ignite.ci.github.ignited.IGitHubConnIgnitedProvider; -import org.apache.ignite.ci.jira.ignited.IJiraIgnited; -import org.apache.ignite.ci.jira.ignited.IJiraIgnitedProvider; -import org.apache.ignite.ci.jira.pure.Ticket; +import org.apache.ignite.githubignited.IGitHubConnIgnited; +import org.apache.ignite.githubignited.IGitHubConnIgnitedProvider; +import org.apache.ignite.jiraignited.IJiraIgnited; +import org.apache.ignite.jiraignited.IJiraIgnitedProvider; +import org.apache.ignite.jiraservice.Ticket; import org.apache.ignite.ci.observer.BuildObserver; import org.apache.ignite.ci.observer.BuildsInfo; import org.apache.ignite.ci.tcbot.ITcBotBgAuth; -import org.apache.ignite.ci.tcbot.chain.PrChainsProcessor; +import org.apache.ignite.tcbot.engine.pr.PrChainsProcessor; import org.apache.ignite.tcbot.common.conf.IGitHubConfig; import org.apache.ignite.tcbot.common.conf.IJiraServerConfig; import org.apache.ignite.tcbot.engine.conf.ITcBotConfig; @@ -130,7 +132,8 @@ public class TcBotTriggerAndSignOffService { /** Config. */ @Inject ITcBotConfig cfg; - @Inject BranchTicketMatcher ticketMatcher; + @Inject + BranchTicketMatcher ticketMatcher; /** Jackson serializer. */ private final ObjectMapper objMapper = new ObjectMapper(); diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java index 7457916ce..d284a4673 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java @@ -28,7 +28,7 @@ public static final String GITHUB_REF = "https://github.com/apache/ignite-teamcity-bot"; /** TC Bot Version. */ - public static final String VERSION = "20190607"; + public static final String VERSION = "20190704"; /** Java version, where Web App is running. */ public String javaVer; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/TriggerBuilds.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/TriggerBuilds.java index 54de527ca..b2466ca89 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/TriggerBuilds.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/TriggerBuilds.java @@ -31,8 +31,9 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; -import org.apache.ignite.ci.github.pure.IGitHubConnection; -import org.apache.ignite.ci.github.pure.IGitHubConnectionProvider; + +import org.apache.ignite.githubservice.IGitHubConnection; +import org.apache.ignite.githubservice.IGitHubConnectionProvider; import org.apache.ignite.tcbot.common.conf.IJiraServerConfig; import org.apache.ignite.tcbot.engine.conf.ITcBotConfig; import org.apache.ignite.ci.tcbot.trigger.TriggerResult; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/buildtime/BuildTimeRestService.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/buildtime/BuildTimeRestService.java new file mode 100644 index 000000000..88986aa76 --- /dev/null +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/buildtime/BuildTimeRestService.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ci.web.rest.buildtime; + +import javax.servlet.ServletContext; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import org.apache.ignite.ci.user.ITcBotUserCreds; +import org.apache.ignite.ci.web.CtxListener; +import org.apache.ignite.tcbot.engine.buildtime.BuildTimeService; +import org.apache.ignite.tcbot.engine.ui.BuildTimeResultUi; +import org.jetbrains.annotations.Nullable; + +@Path(BuildTimeRestService.LONG_RUNNING_SUMMARY) +@Produces(MediaType.APPLICATION_JSON) +public class BuildTimeRestService { + public static final String LONG_RUNNING_SUMMARY = "buildtime"; + + /** Servlet Context. */ + @Context + private ServletContext ctx; + + /** Current Request. */ + @Context + private HttpServletRequest req; + + @GET + @Path("analytics") + public BuildTimeResultUi loadAnalytics(@Nullable @QueryParam("branch") String branchOrNull) { + final ITcBotUserCreds creds = ITcBotUserCreds.get(req); + + final BuildTimeService tbProc = CtxListener.getInjector(ctx).getInstance(BuildTimeService.class); + + return tbProc.analytics( creds); + } +} diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/parms/FullQueryParams.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/parms/FullQueryParams.java deleted file mode 100644 index 11d0b56f2..000000000 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/parms/FullQueryParams.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.ci.web.rest.parms; - -import com.google.common.base.MoreObjects; -import java.util.Objects; -import javax.annotation.Nonnull; -import javax.ws.rs.QueryParam; -import org.jetbrains.annotations.Nullable; - -/** - * Contains maximum combination of query parameters - */ -public class FullQueryParams { - - //see definitions in Index.html javascript - public static final String HISTORY = "History"; - public static final String LATEST = "Latest"; - public static final String CHAIN = "Chain"; - public static final int DEFAULT_COUNT = 10; - - /** Tracked branch name (branch naming in terms of bot). */ - @Nullable @QueryParam("branch") String branch; - - /** Server For not tracked branches. */ - @Nullable @QueryParam("serverId") String serverId; - - /** Suite ID to check. For not tracked branches. */ - @Nonnull @QueryParam("suiteId") String suiteId; - - /** TC identified branch. For not tracked branches. */ - @Nonnull @QueryParam("branchForTc") String branchForTc; - - /** Type of rebuilds loading: */ - @Nonnull @QueryParam("action") String action; - - /** Count of suites to analyze, for multiple runs results. */ - @Nullable @QueryParam("count") Integer count; - - /** Enables all logs to be loaded locally without relation to run/failure category. */ - @Nullable @QueryParam("checkAllLogs") Boolean checkAllLogs; - private Integer buildId; - - /** TC identified base branch: null means the same as <default>, master. For not tracked branches. */ - @Nullable @QueryParam("baseBranchForTc") private String baseBranchForTc; - - /** TC project identifier */ - @Nullable @QueryParam("projectId") String projectId; - - /** TC test name */ - @Nullable @QueryParam("testName") String testName; - - public FullQueryParams() { - } - - public FullQueryParams(String serverId, String suiteId, String branchForTc, String action, Integer count, - String baseBranchForTc) { - this.serverId = serverId; - this.suiteId = suiteId; - this.branchForTc = branchForTc; - this.action = action; - this.count = count; - this.baseBranchForTc = baseBranchForTc; - } - - @Nullable public String getBranch() { - return branch; - } - - @Nullable public String getServerId() { - return serverId; - } - - @Nonnull public String getSuiteId() { - return suiteId; - } - - @Nonnull public String getBranchForTc() { - return branchForTc; - } - - @Nonnull public String getAction() { - return action; - } - - @Nullable public Integer getCount() { - return count; - } - - @Nullable public String getTestName() { - return testName; - } - - @Nullable public String getProjectId() { - return projectId ; - } - - @Nullable public Boolean getCheckAllLogs() { - return checkAllLogs; - } - - /** {@inheritDoc} */ - @Override public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - - FullQueryParams params = (FullQueryParams)o; - - return Objects.equals(branch, params.branch) && - Objects.equals(serverId, params.serverId) && - Objects.equals(suiteId, params.suiteId) && - Objects.equals(branchForTc, params.branchForTc) && - Objects.equals(action, params.action) && - Objects.equals(count, params.count) && - Objects.equals(checkAllLogs, params.checkAllLogs) && - Objects.equals(buildId, params.buildId) && - Objects.equals(baseBranchForTc, params.baseBranchForTc) && - Objects.equals(projectId, params.projectId) && - Objects.equals(testName, params.testName); - } - - /** {@inheritDoc} */ - @Override public int hashCode() { - return Objects.hash(branch, serverId, suiteId, branchForTc, action, count, checkAllLogs, buildId, - baseBranchForTc, projectId, testName); - } - - public void setBranch(@Nullable String branch) { - this.branch = branch; - } - - public void setCheckAllLogs(@Nullable Boolean checkAllLogs) { - this.checkAllLogs = checkAllLogs; - } - - @Override public String toString() { - return MoreObjects.toStringHelper(this) - .add("branch", branch) - .add("serverId", serverId) - .add("suiteId", suiteId) - .add("branchForTc", branchForTc) - .add("action", action) - .add("count", count) - .add("checkAllLogs", checkAllLogs) - .add("buildId", buildId) - .add("baseBranchForTc", baseBranchForTc) - .add("projectId", projectId) - .add("testName", testName) - .toString(); - } - - public void setSuiteId(@Nonnull String suiteId) { - this.suiteId = suiteId; - } - - public void setCount(@Nullable int count) { - this.count = count; - } - - public void setProjectId(@Nullable String projectId) { - this.projectId = projectId; - } - - public void setTestName(@Nullable String testName) { - this.testName = testName; - } - - public void setBuildId(Integer buildId) { - this.buildId = buildId; - } - - public void setServerId(@Nullable String serverId) { - this.serverId = serverId; - } - - public Integer getBuildId() { - return buildId; - } -} diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/pr/GetPrTestFailures.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/pr/GetPrTestFailures.java index 700b3f2e7..644a195ad 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/pr/GetPrTestFailures.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/pr/GetPrTestFailures.java @@ -30,10 +30,10 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import org.apache.ignite.ci.github.PullRequest; -import org.apache.ignite.ci.github.ignited.IGitHubConnIgnited; -import org.apache.ignite.ci.github.ignited.IGitHubConnIgnitedProvider; -import org.apache.ignite.ci.github.pure.IGitHubConnection; -import org.apache.ignite.ci.tcbot.chain.PrChainsProcessor; +import org.apache.ignite.githubignited.IGitHubConnIgnited; +import org.apache.ignite.githubignited.IGitHubConnIgnitedProvider; +import org.apache.ignite.tcbot.engine.pr.PrChainsProcessor; +import org.apache.ignite.githubservice.IGitHubConnection; import org.apache.ignite.tcignited.SyncMode; import org.apache.ignite.ci.user.ITcBotUserCreds; import org.apache.ignite.ci.web.CtxListener; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java index 5d8cce12d..e0fbc41fb 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java @@ -34,7 +34,6 @@ import org.apache.ignite.ci.web.CtxListener; import org.apache.ignite.tcbot.engine.ui.DsSummaryUi; import org.apache.ignite.tcbot.engine.ui.UpdateInfo; -import org.apache.ignite.ci.web.rest.parms.FullQueryParams; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.tcignited.ITeamcityIgnitedProvider; import org.apache.ignite.tcignited.SyncMode; @@ -48,6 +47,7 @@ @Produces(MediaType.APPLICATION_JSON) public class GetTrackedBranchTestResults { public static final String TRACKED = "tracked"; + public static final int DEFAULT_COUNT = 10; /** Servlet Context. */ @Context @@ -138,7 +138,7 @@ public DsSummaryUi getAllTestFailsForMergedBuidls(@Nullable @QueryParam("branch" @QueryParam("checkAllLogs") @Nullable Boolean checkAllLogs, SyncMode mode) { ITcBotUserCreds creds = ITcBotUserCreds.get(req); - int cntLimit = cnt == null ? FullQueryParams.DEFAULT_COUNT : cnt; + int cntLimit = cnt == null ? DEFAULT_COUNT : cnt; Injector injector = CtxListener.getInjector(ctx); return injector.getInstance(TrackedBranchChainsProcessor.class) diff --git a/ignite-tc-helper-web/src/main/webapp/all.html b/ignite-tc-helper-web/src/main/webapp/all.html index e6dd691cf..c3ac6edaa 100644 --- a/ignite-tc-helper-web/src/main/webapp/all.html +++ b/ignite-tc-helper-web/src/main/webapp/all.html @@ -27,6 +27,17 @@ //todo fix setInterval( function() { checkForUpdate(); }, 60000); $.ajax({url: "rest/branches/version", success: showVersionInfo, error: showErrInLoadStatus}); + + $( "#tagSelect" ).change(function() { + var selectedTag = $( "#tagSelect option:selected" ).val() + + $( ".suiteBlock" ).each(function() { + if(selectedTag == 'all' || $(this).find(".buildTag:contains('" + selectedTag +"')").length > 0) + $(this).show(); + else + $(this).hide(); + }) + }); }); @@ -135,12 +146,41 @@ $("#divFailures").html(showChainOnServersResults(result) + " report mode"); + + refillTagSelect(); + } + + function refillTagSelect() { + var newTagList = new Set(); + + $("div#divFailures table tr td span.buildTag").each(function(){ + newTagList.add(this.innerText); + }) + + var tagSelect = $("#tagSelect"); + + tagSelect.empty(); + + tagSelect.append( + $("") + .attr("value", "all") + .text("All") + ); + + newTagList.forEach(function(key) { + tagSelect.append( + $("") + .attr("value", key) + .text(key) + ); + }); }
- + +
diff --git a/ignite-tc-helper-web/src/main/webapp/buildtime.html b/ignite-tc-helper-web/src/main/webapp/buildtime.html new file mode 100644 index 000000000..7090beb56 --- /dev/null +++ b/ignite-tc-helper-web/src/main/webapp/buildtime.html @@ -0,0 +1,139 @@ + + + Apache Ignite Teamcity Bot - Build Time Analytics + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ + + + + + + + + +
Longest average duration (more than 90 minutes)
+ + + + +
Timed out suites average duration (more than 60 minutes)
+ + + + + + + + +
+ +
+ + \ No newline at end of file diff --git a/ignite-tc-helper-web/src/main/webapp/index.html b/ignite-tc-helper-web/src/main/webapp/index.html index e60ab5215..86f12f361 100644 --- a/ignite-tc-helper-web/src/main/webapp/index.html +++ b/ignite-tc-helper-web/src/main/webapp/index.html @@ -37,6 +37,11 @@ }, error: showErrInLoadStatus }); + + $.ajax({ url: "/rest/buildtime/analytics", + success: function (data) { + $("#loadStatus").html(""); + }, error: showErrInLoadStatus }); } diff --git a/ignite-tc-helper-web/src/main/webapp/js/issues-1.0.js b/ignite-tc-helper-web/src/main/webapp/js/issues-1.0.js index decbf503c..1a282be82 100644 --- a/ignite-tc-helper-web/src/main/webapp/js/issues-1.0.js +++ b/ignite-tc-helper-web/src/main/webapp/js/issues-1.0.js @@ -19,20 +19,25 @@ function showIssues(result) { var res = ""; res += "Build problems"; - res += "
"; if (!isDefinedAndFilled(result.issues)) { return res; } + res += " " + result.issues.length; + + res += ":
"; + for (var i = 0; i < result.issues.length; i++) { var issue = result.issues[i]; var color = 'red'; var issueTitle = ''; - res += " "; + //res += " "; - res += issue.displayType; + res += "• "; + + res += issue.type; res += " " + issue.issueKey.testOrBuildName; @@ -46,6 +51,19 @@ function showIssues(result) { } res += "]"; } + res += " tbr=" + issue.trackedBranchName + " bID=" + issue.issueKey.buildId; + + if(isDefinedAndFilled(issue.stat)) { + res += " stat="; + res += JSON.stringify(issue.stat); + } + + if(isDefinedAndFilled(issue.notificationFailed)) { + res += " notificationFailed="; + res += JSON.stringify(issue.notificationFailed); + } + + res += " retry=" + issue.notificationRetry; res += "

"; } diff --git a/ignite-tc-helper-web/src/main/webapp/js/testfails-2.2.js b/ignite-tc-helper-web/src/main/webapp/js/testfails-2.2.js index ca60cfcb4..b94e18143 100644 --- a/ignite-tc-helper-web/src/main/webapp/js/testfails-2.2.js +++ b/ignite-tc-helper-web/src/main/webapp/js/testfails-2.2.js @@ -588,8 +588,10 @@ function showSuiteData(suite, settings, prNum) { moreInfoTxt += "
"; - var res = ""; - res += ""; + var res = "" + + "" + + res += ""; + res += "
"; var failRateText = ""; if (isDefinedAndFilled(suite.failures) && isDefinedAndFilled(suite.runs) && isDefinedAndFilled(suite.failureRate)) { @@ -620,7 +622,7 @@ function showSuiteData(suite, settings, prNum) { res += drawLatestRuns(suite.latestRuns) + " "; } - res +=""; + res +=""; res += " "; res += "" + suite.name + " " + @@ -729,6 +731,8 @@ function showSuiteData(suite, settings, prNum) { res += "
  
" + return res; } @@ -780,8 +784,7 @@ function showTestFailData(testFail, isFailureShown, settings) { return ""; // test is hidder } - var res = ""; - res += ""; + var res = ""; var haveIssue = isDefinedAndFilled(testFail.webIssueUrl) && isDefinedAndFilled(testFail.webIssueText); @@ -940,8 +943,7 @@ function showTestFailData(testFail, isFailureShown, settings) { } - - res += ""; + res += ""; var haveBaseBranchWeb = isDefinedAndFilled(testFail.webUrlBaseBranch); if (haveBaseBranchWeb) diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/MockBasedTcBotModule.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/MockBasedTcBotModule.java index 8755a5939..554b93011 100644 --- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/MockBasedTcBotModule.java +++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/MockBasedTcBotModule.java @@ -23,15 +23,15 @@ import java.util.Properties; import org.apache.ignite.ci.HelperConfig; import org.apache.ignite.ci.github.PullRequest; -import org.apache.ignite.ci.github.ignited.IGitHubConnIgnited; -import org.apache.ignite.ci.github.ignited.IGitHubConnIgnitedProvider; -import org.apache.ignite.ci.github.pure.IGitHubConnection; -import org.apache.ignite.ci.github.pure.IGitHubConnectionProvider; -import org.apache.ignite.ci.jira.ignited.IJiraIgnited; -import org.apache.ignite.ci.jira.ignited.IJiraIgnitedProvider; -import org.apache.ignite.ci.jira.pure.IJiraIntegration; -import org.apache.ignite.ci.jira.pure.IJiraIntegrationProvider; +import org.apache.ignite.githubignited.IGitHubConnIgnited; +import org.apache.ignite.githubignited.IGitHubConnIgnitedProvider; +import org.apache.ignite.jiraignited.IJiraIgnited; +import org.apache.ignite.jiraignited.IJiraIgnitedProvider; +import org.apache.ignite.jiraservice.IJiraIntegration; +import org.apache.ignite.jiraservice.IJiraIntegrationProvider; import org.apache.ignite.ci.tcbot.conf.TcBotJsonConfig; +import org.apache.ignite.githubservice.IGitHubConnection; +import org.apache.ignite.githubservice.IGitHubConnectionProvider; import org.apache.ignite.tcbot.common.conf.IGitHubConfig; import org.apache.ignite.tcbot.common.conf.IJiraServerConfig; import org.apache.ignite.tcbot.engine.conf.ITcBotConfig; diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessorTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessorTest.java index 2dec78103..d4e93f631 100644 --- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessorTest.java +++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessorTest.java @@ -27,6 +27,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.function.Predicate; + +import org.apache.ignite.tcbot.common.TcBotConst; +import org.apache.ignite.tcbot.engine.pr.PrChainsProcessor; import org.apache.ignite.tcservice.ITeamcity; import org.apache.ignite.tcservice.model.conf.BuildType; import org.apache.ignite.tcservice.model.hist.BuildRef; @@ -220,7 +223,7 @@ public Map initHistory(IStringCompactor c) { addBuildsToEmulatedStor(cache1InMaster); } - long ageMs = TimeUnit.DAYS.toMillis(InvocationData.MAX_DAYS); + long ageMs = TimeUnit.DAYS.toMillis(TcBotConst.HISTORY_MAX_DAYS); for (int i = 0; i < 134; i++) { addBuildsToEmulatedStor(createFailedBuild(c, CACHE_1, diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java index 4f9eacddf..0c81bb9a3 100644 --- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java +++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java @@ -31,6 +31,7 @@ import org.apache.ignite.tcbot.persistence.IStringCompactor; import org.apache.ignite.tcignited.ITeamcityIgnited; import org.apache.ignite.tcignited.SyncMode; +import org.apache.ignite.tcignited.history.ISuiteRunHistory; import org.apache.ignite.tcservice.model.result.tests.TestOccurrence; import org.jetbrains.annotations.NotNull; import org.mockito.Mockito; @@ -90,10 +91,10 @@ public static ITeamcityIgnited getMutableMapTeamcityIgnited(Map { - final String name = inv.getArgument(0); - final String branch = inv.getArgument(1); + final Integer tstName = inv.getArgument(0); + final Integer branchId = inv.getArgument(2); // System.out.println("Search history " + name + " in " + branch + ": " ); if (histCache.isEmpty()) { @@ -103,11 +104,9 @@ public static ITeamcityIgnited getMutableMapTeamcityIgnited(Map " + runHistCompacted); + System.out.println("Test history " + c.getStringFromId(tstName) + " in " + c.getStringFromId(branchId) + " => " + runHistCompacted); return runHistCompacted; }); + when(tcIgnited.getSuiteRunHist(anyInt(), anyInt())) + .thenAnswer((inv) -> { + final Integer suiteName = inv.getArgument(0); + final Integer branchId = inv.getArgument(1); + // System.out.println("Search history " + name + " in " + branch + ": " ); + if (histCache.isEmpty()) { + synchronized (histCache) { + if (histCache.isEmpty()) + initHistory(c, histCache, builds, srvId); + } + } + + ISuiteRunHistory mock = Mockito.mock(ISuiteRunHistory.class); + + when(mock.getTestRunHist(anyInt())).thenAnswer((inv2)-> { + final Integer tstName = inv2.getArgument(0); + + if (tstName == null) + return null; + + if (branchId == null) + return null; + + final RunHistKey key = new RunHistKey(srvId, tstName, branchId); + + final RunHistCompacted runHistCompacted = histCache.get(key); + + System.out.println("Test history " + c.getStringFromId(tstName) + " in " + c.getStringFromId(branchId) + " => " + runHistCompacted); + + return runHistCompacted; + }); + + return mock; + }); + // when(tcIgnited.gitBranchPrefix()).thenReturn("ignite-"); ITcServerConfig mock = mock(ITcServerConfig.class); @@ -145,7 +179,7 @@ public static void initHistory(IStringCompactor c, Map true, successStatusStrId); - hist.addInvocation(inv); + hist.innerAddInvocation(inv); }); } diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/tcignited/IgnitedTcInMemoryIntegrationTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/tcignited/IgnitedTcInMemoryIntegrationTest.java index 6afa3801e..398c8f881 100644 --- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/tcignited/IgnitedTcInMemoryIntegrationTest.java +++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/tcignited/IgnitedTcInMemoryIntegrationTest.java @@ -26,7 +26,7 @@ import org.apache.ignite.Ignition; import org.apache.ignite.ci.db.TcHelperDb; import org.apache.ignite.ci.di.scheduler.DirectExecNoWaitScheduler; -import org.apache.ignite.ci.jira.pure.IJiraIntegrationProvider; +import org.apache.ignite.jiraservice.IJiraIntegrationProvider; import org.apache.ignite.ci.tcbot.chain.PrChainsProcessorTest; import org.apache.ignite.ci.tcbot.conf.TcBotJsonConfig; import org.apache.ignite.tcbot.engine.conf.ITcBotConfig; diff --git a/jetty-launcher/build.gradle b/jetty-launcher/build.gradle index 120d87f48..5df3d7b26 100644 --- a/jetty-launcher/build.gradle +++ b/jetty-launcher/build.gradle @@ -18,14 +18,6 @@ apply plugin: 'java' apply plugin: 'application' -sourceCompatibility = '1.8' -targetCompatibility = '1.8' - -repositories { - mavenCentral() - mavenLocal() -} - mainClassName = 'org.apache.ignite.ci.TcHelperJettyLauncher' applicationDefaultJvmArgs = ["-Dteamcity.helper.home=../work", "-Dteamcity.bot.regionsize=16", // 16g Durable Memory region diff --git a/settings.gradle b/settings.gradle index b421c7c3a..423525fcc 100644 --- a/settings.gradle +++ b/settings.gradle @@ -5,5 +5,9 @@ include 'tcbot-common' include 'tcbot-persistence' include 'tcbot-teamcity' include 'tcbot-teamcity-ignited' +include 'tcbot-github' +include 'tcbot-github-ignited' +include 'tcbot-jira' +include 'tcbot-jira-ignited' include 'tcbot-engine' diff --git a/tcbot-common/build.gradle b/tcbot-common/build.gradle index 4b3cb08db..dc1148b33 100644 --- a/tcbot-common/build.gradle +++ b/tcbot-common/build.gradle @@ -17,11 +17,6 @@ apply plugin: 'java' -repositories { - mavenCentral() - mavenLocal() -} - dependencies { compile group: 'com.google.guava', name: 'guava', version: guavaVer @@ -29,16 +24,14 @@ dependencies { //Apache License Version 2.0 compile group: 'com.google.code.gson', name: 'gson', version: gsonVer - testCompile group: 'junit', name: 'junit', version: '4.12' - testCompile group: 'org.mockito', name: 'mockito-core', version: '2.22.0' - compile group: 'javax.inject', name: 'javax.inject', version: '1' compile group: 'com.google.inject', name: 'guice', version: '4.2.0' compile group: 'org.xerial.snappy', name: 'snappy-java', version: '1.1.7.2' - /// JAXB replacement for Java 11 + //impl is used by TC module, only converters needed here compile group: 'javax.xml.bind', name: 'jaxb-api', version: '2.3.1' - compile group: 'com.sun.xml.bind', name: 'jaxb-impl', version: '2.3.0' - compile group: 'com.sun.xml.bind', name: 'jaxb-core', version: '2.3.0' + + testCompile group: 'junit', name: 'junit', version: junitVer + testCompile group: 'org.mockito', name: 'mockito-core', version: mockitoVer } diff --git a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/TcBotConst.java b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/TcBotConst.java new file mode 100644 index 000000000..44433db4e --- /dev/null +++ b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/TcBotConst.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.tcbot.common; + +/** + * TC Bot constants: contains magic numbers for project. + * + * Usually it is a practically set up rules, heuristics used by the bot. + */ +public class TcBotConst { + /** Max days to keep test invocatoin data in run statistics: affects Bot Visa. */ + public static final int HISTORY_MAX_DAYS = 21; + + /** Absulte Max days to keep track about build existence, 42 */ + public static final int BUILD_MAX_DAYS = HISTORY_MAX_DAYS * 2; + + /** History collection process: build id per server ID border days. */ + public static final int HISTORY_BUILD_ID_BORDER_DAYS = HISTORY_MAX_DAYS + 2; + + /** Notify about failure: max build age days (since build start time). */ + public static final int NOTIFY_MAX_AGE_SINCE_START_DAYS = HISTORY_MAX_DAYS / 2; + + /** */ + public static final int NOTIFY_MAX_AGE_SINCE_DETECT_HOURS = 2; + + /** */ + public static final int NOTIFY_MAX_AGE_SINCE_DETECT_FOR_NOTIFIED_ISSUE_HOURS = 24; + + /** Flakyness status change border: Count of test change status before considered as flaky. */ + public static final int FLAKYNESS_STATUS_CHANGE_BORDER = 1; + + /** Non flaky test failure rate: less that this failure rate in base branch is still blocker border, percents. */ + public static final double NON_FLAKY_TEST_FAIL_RATE_BLOCKER_BORDER_PERCENTS = 4.; +} diff --git a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/conf/IDataSourcesConfigSupplier.java b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/conf/IDataSourcesConfigSupplier.java index 9ef5b1ffc..68873ad65 100644 --- a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/conf/IDataSourcesConfigSupplier.java +++ b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/conf/IDataSourcesConfigSupplier.java @@ -22,4 +22,8 @@ */ public interface IDataSourcesConfigSupplier { public ITcServerConfig getTeamcityConfig(String srvCode); + + public IGitHubConfig getGitConfig(String srvCode); + + public IJiraServerConfig getJiraConfig(String srvCode); } diff --git a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/conf/PasswordEncoder.java b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/conf/PasswordEncoder.java index e86801b18..166d888ae 100644 --- a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/conf/PasswordEncoder.java +++ b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/conf/PasswordEncoder.java @@ -19,14 +19,13 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; -import org.apache.ignite.tcbot.common.util.Base64Util; -import org.apache.ignite.tcbot.common.util.CryptUtil; - import java.security.SecureRandom; import javax.annotation.Nonnull; import javax.crypto.Cipher; import javax.crypto.spec.SecretKeySpec; import javax.xml.bind.DatatypeConverter; +import org.apache.ignite.tcbot.common.util.Base64Util; +import org.apache.ignite.tcbot.common.util.CryptUtil; import static javax.xml.bind.DatatypeConverter.parseHexBinary; import static javax.xml.bind.DatatypeConverter.printHexBinary; @@ -118,7 +117,7 @@ public static void main(String[] args) { } public static void main1(String[] args) { - encodeJiraTok("ignitetcbot", "21313"); + encodeJiraTok("ignitetcbot", "enterClearPasswordOrTokenForUser"); } public static void encodeJiraTok(String user, String pwd) { diff --git a/tcbot-engine/build.gradle b/tcbot-engine/build.gradle index 411db8dcc..7050ae2a4 100644 --- a/tcbot-engine/build.gradle +++ b/tcbot-engine/build.gradle @@ -17,13 +17,9 @@ apply plugin: 'java' -repositories { - mavenCentral() - mavenLocal() -} - dependencies { - compile (project(":tcbot-persistence")); - compile (project(":tcbot-teamcity-ignited")); + compile (project(":tcbot-teamcity-ignited")); + compile (project(":tcbot-github-ignited")); + compile (project(":tcbot-jira-ignited")); } \ No newline at end of file diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java new file mode 100644 index 000000000..6e4775714 --- /dev/null +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.tcbot.engine.buildtime; + +import org.apache.ignite.tcbot.common.interceptor.MonitoredTask; +import org.apache.ignite.tcbot.common.util.TimeUtil; +import org.apache.ignite.tcbot.engine.conf.ITcBotConfig; +import org.apache.ignite.tcbot.engine.ui.BuildTimeRecordUi; +import org.apache.ignite.tcbot.engine.ui.BuildTimeResultUi; +import org.apache.ignite.tcbot.persistence.IStringCompactor; +import org.apache.ignite.tcbot.persistence.scheduler.IScheduler; +import org.apache.ignite.tcignited.ITeamcityIgnited; +import org.apache.ignite.tcignited.ITeamcityIgnitedProvider; +import org.apache.ignite.tcignited.build.FatBuildDao; +import org.apache.ignite.tcignited.buildref.BuildRefDao; +import org.apache.ignite.tcignited.buildtime.BuildTimeRecord; +import org.apache.ignite.tcignited.buildtime.BuildTimeResult; +import org.apache.ignite.tcignited.creds.ICredentialsProv; +import org.apache.ignite.tcignited.history.HistoryCollector; + +import javax.inject.Inject; +import java.time.Duration; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +public class BuildTimeService { + @Inject ITeamcityIgnitedProvider tcProv; + + /** Config. */ + @Inject ITcBotConfig cfg; + + @Inject FatBuildDao fatBuildDao; + + @Inject IStringCompactor compactor; + + @Inject HistoryCollector historyCollector; + + @Inject IScheduler scheduler; + + private volatile BuildTimeResult lastRes1d = new BuildTimeResult(); + + @Inject private BuildRefDao buildRefDao; + + public BuildTimeResultUi analytics(ICredentialsProv prov) { + if (buildRefDao.buildRefsCache() == null) + return new BuildTimeResultUi(); + + Collection allServers = cfg.getServerIds(); + + scheduler.sheduleNamed("BuildTimeService.loadAnalytics", + this::loadAnalytics, 15, TimeUnit.MINUTES); + + Set availableServers = allServers.stream() + .filter(prov::hasAccess) + .map(ITeamcityIgnited::serverIdToInt) + .collect(Collectors.toSet()); + + BuildTimeResultUi resultUi = new BuildTimeResultUi(); + + long minDuration = Duration.ofMinutes(90).toMillis(); + long minDurationTimeout = Duration.ofMinutes(60).toMillis(); + int cntToInclude = 50; + BuildTimeResult res = lastRes1d; + + res.topByBuildTypes(availableServers, minDuration, cntToInclude) + .stream().map(this::convertToUi).forEach(e -> resultUi.byBuildType.add(e)); + + res.topTimeoutsByBuildTypes(availableServers, minDurationTimeout, cntToInclude) + .stream().map(this::convertToUi).forEach(e -> resultUi.timedOutByBuildType.add(e)); + + return resultUi; + } + + public BuildTimeRecordUi convertToUi(Map.Entry e) { + BuildTimeRecordUi buildTimeRecordUi = new BuildTimeRecordUi(); + Long key = e.getKey(); + int btId = BuildTimeResult.cacheKeyToBuildType(key); + buildTimeRecordUi.buildType = compactor.getStringFromId(btId); + + buildTimeRecordUi.averageDuration = TimeUtil.millisToDurationPrintable(e.getValue().avgDuration()); + buildTimeRecordUi.totalDuration = TimeUtil.millisToDurationPrintable(e.getValue().totalDuration()); + return buildTimeRecordUi; + } + + @SuppressWarnings("WeakerAccess") + @MonitoredTask(name = "Load Build Time Analytics") + protected void loadAnalytics() { + int days = 1; + + List idsToCheck = historyCollector.findAllRecentBuilds(days, cfg.getServerIds()); + + BuildTimeResult res = fatBuildDao.loadBuildTimeResult(days, idsToCheck); + + lastRes1d = res; + } +} diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java index 62bb20946..579dbb0a3 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java @@ -19,12 +19,32 @@ import com.google.common.base.Preconditions; import com.google.common.util.concurrent.Futures; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Future; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.inject.Inject; import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted; import org.apache.ignite.ci.teamcity.ignited.buildtype.ParametersCompacted; import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; +import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; import org.apache.ignite.tcbot.common.util.FutureUtil; import org.apache.ignite.tcbot.engine.pool.TcUpdatePool; -import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; import org.apache.ignite.tcbot.engine.ui.LrTestUi; import org.apache.ignite.tcbot.engine.ui.LrTestsSuiteSummaryUi; import org.apache.ignite.tcbot.persistence.IStringCompactor; @@ -38,18 +58,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import javax.inject.Inject; -import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Future; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; - /** * Process whole Build Chain, E.g. runAll at particular server, including all builds involved */ @@ -144,6 +152,8 @@ public FullChainRunCtx loadFullChainContext( if (entryPoints.isEmpty()) return new FullChainRunCtx(Build.createFakeStub()); + Integer failRateBranchId = compactor.getStringIdIfPresent(RunHistSync.normalizeBranch(failRateBranch)); + Map> builds = loadAllBuildsInChains(entryPoints, mode, tcIgn); Map>> freshRebuilds = new ConcurrentHashMap<>(); @@ -177,6 +187,11 @@ public FullChainRunCtx loadFullChainContext( buildsForSuite.forEach(buildCompacted -> ctx.addBuild(loadChanges(buildCompacted, tcIgn))); + //ask for history for the suite in parallel + tcUpdatePool.getService().submit(() -> { + ctx.history(tcIgn, failRateBranchId); + }); + analyzeTests(ctx, tcIgn, procLog); fillBuildCounts(ctx, tcIgn, includeScheduledInfo); @@ -185,9 +200,7 @@ public FullChainRunCtx loadFullChainContext( }); Function function = ctx -> { - - //todo cache RunStat instance into suite context to compare - IRunHistory runStat = tcIgn.getSuiteRunHist(ctx.suiteId(), RunHistSync.normalizeBranch(failRateBranch)); + IRunHistory runStat = ctx.history(tcIgn, failRateBranchId); if (runStat == null) return 0f; @@ -230,7 +243,8 @@ public Map> loadAllBuildsInChains(Collection< .peek(val -> Preconditions.checkNotNull(val, "Build future should be in context")) .flatMap(ref -> dependencies(ref, mode, builds, tcIgn).stream()).collect(Collectors.toSet()); - logger.info("Level [" + level + "] dependencies:" + depsNextLevel); + if(logger.isDebugEnabled()) + logger.debug("Level [" + level + "] dependencies:" + depsNextLevel); remainedUnloadedDeps = depsNextLevel; } @@ -330,7 +344,7 @@ protected List> replaceWithRecent(List runAllBuilds = teamcityIgnited.getAllBuildsCompacted(outCtx.buildTypeId(), outCtx.branchName()); + List runAllBuilds = teamcityIgnited.getAllBuildsCompacted(outCtx.suiteId(), outCtx.branchName()); long cntRunning = runAllBuilds .stream() diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java index a79411065..f965db3a2 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java @@ -17,7 +17,9 @@ package org.apache.ignite.tcbot.engine.chain; +import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import com.google.common.cache.CacheBuilder; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; @@ -28,23 +30,25 @@ import java.util.OptionalDouble; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; - import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted; import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProblemCompacted; import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted; import org.apache.ignite.tcbot.common.conf.ITcServerConfig; +import org.apache.ignite.tcbot.common.exeption.ExceptionUtil; import org.apache.ignite.tcbot.common.util.CollectionUtil; import org.apache.ignite.tcbot.persistence.IStringCompactor; import org.apache.ignite.tcignited.ITeamcityIgnited; import org.apache.ignite.tcignited.buildlog.ILogCheckResult; import org.apache.ignite.tcignited.buildlog.ITestLogCheckResult; import org.apache.ignite.tcignited.history.IRunHistory; +import org.apache.ignite.tcignited.history.ISuiteRunHistory; import org.apache.ignite.tcservice.model.hist.BuildRef; import org.apache.ignite.tcservice.model.result.problems.ProblemOccurrence; import org.apache.ignite.tcservice.model.result.stat.Statistics; @@ -67,6 +71,9 @@ public class MultBuildRunCtx implements ISuiteResults { /** Builds: Single execution. */ private List builds = new CopyOnWriteArrayList<>(); + private final com.google.common.cache.Cache> historyCacheMap + = CacheBuilder.newBuilder().build(); + public void addBuild(SingleBuildRunCtx ctx) { builds.add(ctx); } @@ -98,8 +105,9 @@ public Stream> getLogsCheckResults() { return buildsStream().map(SingleBuildRunCtx::getTestLogCheckResult).filter(Objects::nonNull); } - public String suiteId() { - return firstBuildInfo.suiteId(); + /** {@inheritDoc} */ + @Override public String suiteId() { + return firstBuild().map(SingleBuildRunCtx::suiteId).orElse(null); } /** {@inheritDoc} */ @@ -107,9 +115,6 @@ public String suiteId() { return getBuildMessageProblemCount() > 0; } - public String buildTypeId() { - return firstBuildInfo.buildTypeId; - } public boolean hasAnyBuildProblemExceptTestOrSnapshot() { return allProblemsInAllBuilds() @@ -170,6 +175,7 @@ public long getCompilationProblemCount() { return buildsStream().filter(ISuiteResults::hasCompilationProblem).count(); } + /** {@inheritDoc} */ public boolean hasTimeoutProblem() { return getExecutionTimeoutCount() > 0; } @@ -178,6 +184,7 @@ private long getExecutionTimeoutCount() { return buildsStream().filter(SingleBuildRunCtx::hasTimeoutProblem).count(); } + /** {@inheritDoc} */ public boolean hasJvmCrashProblem() { return getJvmCrashProblemCount() > 0; } @@ -186,10 +193,12 @@ public long getJvmCrashProblemCount() { return buildsCntHavingBuildProblem(ProblemOccurrence.TC_JVM_CRASH); } + /** {@inheritDoc} */ public boolean hasOomeProblem() { return getOomeProblemCount() > 0; } + /** {@inheritDoc} */ @Override public boolean hasExitCodeProblem() { return getExitCodeProblemsCount() > 0; } @@ -303,8 +312,8 @@ public Stream> getTopLogConsumers() { return CollectionUtil.top(logSizeBytes.entrySet().stream(), 3, comparing).stream(); } - public Stream getTopLongRunning() { - Comparator comparing = Comparator.comparing(IMultTestOccurrence::getAvgDurationMs); + public Stream getTopLongRunning() { + Comparator comparing = Comparator.comparing(TestCompactedMult::getAvgDurationMs); Map res = new HashMap<>(); @@ -315,7 +324,7 @@ public Stream getTopLongRunning() { return CollectionUtil.top(res.values().stream(), 3, comparing).stream(); } - public List getFailedTests() { + public List getFailedTests() { Map res = new HashMap<>(); builds.forEach(singleBuildRunCtx -> { @@ -327,7 +336,7 @@ public List getFailedTests() { public void saveToMap(Map res, Stream tests) { tests.forEach(testCompacted -> { - res.computeIfAbsent(testCompacted.testName(), k -> new TestCompactedMult(compactor)) + res.computeIfAbsent(testCompacted.testName(), k -> new TestCompactedMult(compactor, this)) .add(testCompacted); }); } @@ -607,17 +616,13 @@ public Integer trustedTests(ITeamcityIgnited tcIgnited, //todo can cache mult occurrences in ctx builds.forEach(singleBuildRunCtx -> { - saveToMap(res, singleBuildRunCtx.getAllTests() - .filter(t -> !t.isIgnoredTest() && !t.isMutedTest())); + saveToMap(res, + singleBuildRunCtx.getAllTests().filter(t -> !t.isIgnoredTest() && !t.isMutedTest())); }); Integer branchName = compactor.getStringIdIfPresent(normalizedBaseBranch); - Integer suiteName = compactor.getStringIdIfPresent( buildTypeId()); - - // res.clear(); //todo enable feature back - //todo can cache fail rate in mult occur - res.keySet().forEach((testNameId) -> { - IRunHistory stat = tcIgnited.getTestRunHist(testNameId, suiteName, branchName); + res.forEach((testNameId, compactedMult) -> { + IRunHistory stat = compactedMult.history(tcIgnited, branchName); String testBlockerComment = TestCompactedMult.getPossibleBlockerComment(stat); boolean b = testBlockerComment != null; if (b) // this test will be considered as blocker if will fail @@ -627,4 +632,46 @@ public Integer trustedTests(ITeamcityIgnited tcIgnited, return trustedCnt.get(); } + /** + * Returns suite name non compacted. + */ + public Integer buildTypeIdId() { + return firstBuild().map(SingleBuildRunCtx::buildTypeIdId).orElse(null); + } + + public Optional firstBuild() { + return builds.stream().findFirst(); + } + + /** + * @param tcIgn Tc ign. + * @param baseBranchId Base branch id. + */ + public IRunHistory history(ITeamcityIgnited tcIgn, Integer baseBranchId) { + if (baseBranchId == null) + return null; + + ISuiteRunHistory suiteHist = suiteHist(tcIgn, baseBranchId); + if (suiteHist == null) + return null; + + return suiteHist.self(); + } + + @Nullable + ISuiteRunHistory suiteHist(ITeamcityIgnited tcIgn, Integer baseBranchId) { + Integer buildTypeIdId = buildTypeIdId(); + Preconditions.checkNotNull(buildTypeIdId, "Build type ID should be filled"); + + try { + return historyCacheMap.get(baseBranchId, + () -> { + return Optional.ofNullable(tcIgn.getSuiteRunHist(buildTypeIdId, baseBranchId)); + }) + .orElse(null); + } + catch (ExecutionException e) { + throw ExceptionUtil.propagateException(e); + } + } } diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/SingleBuildRunCtx.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/SingleBuildRunCtx.java index dab646e3f..e41d65f97 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/SingleBuildRunCtx.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/SingleBuildRunCtx.java @@ -346,4 +346,8 @@ public boolean hasSuiteIncompleteFailure() { public int totalNotMutedTests() { return buildCompacted.totalNotMutedTests(); } + + public int buildTypeIdId() { + return buildCompacted.buildTypeId(); + } } diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java index 908bc5f28..621c7cd5e 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java @@ -21,10 +21,15 @@ import java.util.List; import java.util.Objects; import java.util.stream.Collectors; +import javax.annotation.Nullable; import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted; +import org.apache.ignite.tcbot.common.TcBotConst; import org.apache.ignite.tcbot.persistence.IStringCompactor; +import org.apache.ignite.tcignited.ITeamcityIgnited; import org.apache.ignite.tcignited.history.IRunHistSummary; +import org.apache.ignite.tcignited.history.IRunHistory; import org.apache.ignite.tcignited.history.IRunStat; +import org.apache.ignite.tcignited.history.ISuiteRunHistory; import org.apache.ignite.tcservice.model.result.tests.TestOccurrenceFull; /** @@ -33,12 +38,17 @@ public class TestCompactedMult implements IMultTestOccurrence { private final List occurrences = new ArrayList<>(); private IStringCompactor compactor; + private MultBuildRunCtx ctx; private long avgDuration = -1; - public TestCompactedMult(IStringCompactor compactor) { + public TestCompactedMult(IStringCompactor compactor, MultBuildRunCtx ctx) { this.compactor = compactor; + this.ctx = ctx; } + @Nullable public Integer testName() { + return occurrences.isEmpty() ? null : occurrences.iterator().next().testName(); + } /** {@inheritDoc} */ @Override public String getName() { return occurrences.isEmpty() ? "" : occurrences.iterator().next().testName(compactor); @@ -93,7 +103,7 @@ public static String getPossibleBlockerComment(IRunHistSummary baseBranchStat) { boolean flaky = baseBranchStat.isFlaky(); float failRate = baseBranchStat.getFailRate(); - boolean lowFailureRate = failRate * 100.0f < 4.; + boolean lowFailureRate = failRate * 100.0f < TcBotConst.NON_FLAKY_TEST_FAIL_RATE_BLOCKER_BORDER_PERCENTS; if (lowFailureRate && !flaky) { String runStatPrintable = IRunStat.getPercentPrintable(failRate * 100.0f); @@ -109,4 +119,18 @@ public static String getPossibleBlockerComment(IRunHistSummary baseBranchStat) { public void add(TestCompacted next) { occurrences.add(next); } + + + public IRunHistory history(ITeamcityIgnited ignited, Integer baseBranchId) { + Integer name = testName(); + if (name == null) + return null; + + ISuiteRunHistory suiteRunHist = ctx.suiteHist(ignited, baseBranchId); + + if (suiteRunHist == null) + return null; + + return suiteRunHist.getTestRunHist(name); + } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/BranchTicketMatcher.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/BranchTicketMatcher.java similarity index 92% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/BranchTicketMatcher.java rename to tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/BranchTicketMatcher.java index 390e71b7a..996da7118 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/BranchTicketMatcher.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/BranchTicketMatcher.java @@ -15,24 +15,22 @@ * limitations under the License. */ -package org.apache.ignite.ci.tcbot.visa; +package org.apache.ignite.tcbot.engine.pr; import com.google.common.base.Strings; +import org.apache.ignite.githubservice.IGitHubConnection; import org.apache.ignite.tcbot.common.interceptor.GuavaCached; import org.apache.ignite.ci.github.PullRequest; -import org.apache.ignite.ci.github.ignited.IGitHubConnIgnitedProvider; -import org.apache.ignite.ci.github.pure.IGitHubConnection; -import org.apache.ignite.ci.jira.ignited.IJiraIgnitedProvider; -import org.apache.ignite.ci.jira.ignited.TicketCompacted; -import org.apache.ignite.ci.jira.pure.Ticket; +import org.apache.ignite.githubignited.IGitHubConnIgnitedProvider; +import org.apache.ignite.jiraignited.IJiraIgnitedProvider; +import org.apache.ignite.jiraservice.Ticket; import org.apache.ignite.tcbot.common.conf.IGitHubConfig; import org.apache.ignite.tcbot.common.conf.IJiraServerConfig; import org.apache.ignite.tcbot.engine.conf.ITcBotConfig; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import javax.inject.Inject; -import javax.ws.rs.QueryParam; import java.util.Collection; import java.util.Objects; @@ -50,8 +48,8 @@ public class BranchTicketMatcher { @Inject private IJiraIgnitedProvider jiraIgnProv; @Nullable public String resolveTcBranchForPrLess(Ticket ticket, - IJiraServerConfig jiraCfg, - IGitHubConfig gitHubCfg) { + IJiraServerConfig jiraCfg, + IGitHubConfig gitHubCfg) { String branchNumPrefix = jiraCfg.branchNumPrefix(); if (Strings.isNullOrEmpty(branchNumPrefix)) { @@ -98,7 +96,7 @@ private String convertJiraToGit(String branchJiraIdentification, if (Strings.isNullOrEmpty(branchNumPrefix)) { //an easy way, no special branch and ticket mappings specified, use project code. - String jiraPrefix = jiraCfg.projectCodeForVisa() + TicketCompacted.PROJECT_DELIM; + String jiraPrefix = jiraCfg.projectCodeForVisa() + Ticket.PROJECT_DELIM; final String ticketKey = findFixPrefixedNumber(pr.getTitle(), jiraPrefix); @@ -173,7 +171,7 @@ private boolean mentionsBranch(String branchName, Ticket ticket) { return false; } - @Nullable private String findFixPrefixedNoInValues(@NotNull String prefix, String... values) { + @Nullable private String findFixPrefixedNoInValues(@Nonnull String prefix, String... values) { for (String value : values) { String fixPrefixedNum = findFixPrefixedNumber(value, prefix); @@ -188,7 +186,7 @@ private boolean mentionsBranch(String branchName, Ticket ticket) { * @param prefix Ticket prefix. * @return Branch number or null. */ - @Nullable private String findFixPrefixedNumber(@Nullable String val, @NotNull String prefix) { + @Nullable private String findFixPrefixedNumber(@Nullable String val, @Nonnull String prefix) { if (Strings.isNullOrEmpty(val)) return null; @@ -237,7 +235,7 @@ public String resolveTicketFromBranch(String srvCode, String ticketFullName, String branchNumPrefix = jiraCfg.branchNumPrefix(); ticketPrefix = Strings.isNullOrEmpty(branchNumPrefix) - ? jiraCfg.projectCodeForVisa() + TicketCompacted.PROJECT_DELIM + ? jiraCfg.projectCodeForVisa() + Ticket.PROJECT_DELIM : branchNumPrefix; String prLessTicket = prLessTicket(branchForTc, ticketPrefix, gitCfg); @@ -291,9 +289,10 @@ public String resolveTicketFromBranch(String srvCode, String ticketFullName, " or use branch name according ticket name."); } - @Nullable private PullRequest findPrForBranch( - @Nullable @QueryParam("serverId") String srvId, - @Nullable @QueryParam("branchName") String branchForTc) { + @Nullable + private PullRequest findPrForBranch( + @Nullable String srvId, + @Nullable String branchForTc) { Integer prId = IGitHubConnection.convertBranchToPrId(branchForTc); if (prId == null) diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java similarity index 89% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessor.java rename to tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java index a1412866f..5a9246cb4 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessor.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java @@ -14,43 +14,47 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ignite.ci.tcbot.chain; +package org.apache.ignite.tcbot.engine.pr; import com.google.common.base.Strings; import java.util.List; import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import javax.annotation.Nullable; import javax.inject.Inject; import org.apache.ignite.ci.github.PullRequest; -import org.apache.ignite.ci.github.pure.IGitHubConnection; +import org.apache.ignite.githubservice.IGitHubConnection; import org.apache.ignite.tcbot.engine.chain.*; -import org.apache.ignite.ci.github.ignited.IGitHubConnIgnited; -import org.apache.ignite.ci.github.ignited.IGitHubConnIgnitedProvider; -import org.apache.ignite.ci.jira.ignited.IJiraIgnited; -import org.apache.ignite.ci.jira.ignited.IJiraIgnitedProvider; -import org.apache.ignite.ci.tcbot.visa.BranchTicketMatcher; -import org.apache.ignite.ci.user.ITcBotUserCreds; +import org.apache.ignite.githubignited.IGitHubConnIgnited; +import org.apache.ignite.githubignited.IGitHubConnIgnitedProvider; +import org.apache.ignite.jiraignited.IJiraIgnited; +import org.apache.ignite.jiraignited.IJiraIgnitedProvider; import org.apache.ignite.tcbot.engine.ui.DsChainUi; import org.apache.ignite.tcbot.engine.ui.DsSuiteUi; import org.apache.ignite.tcbot.engine.ui.DsTestFailureUi; import org.apache.ignite.tcbot.engine.ui.DsSummaryUi; -import org.apache.ignite.ci.web.rest.parms.FullQueryParams; import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; import org.apache.ignite.tcbot.persistence.IStringCompactor; import org.apache.ignite.tcignited.ITeamcityIgnited; import org.apache.ignite.tcignited.ITeamcityIgnitedProvider; import org.apache.ignite.tcignited.SyncMode; +import org.apache.ignite.tcignited.creds.ICredentialsProv; import org.apache.ignite.tcignited.history.IRunHistory; import org.apache.ignite.tcignited.history.RunHistSync; import org.apache.ignite.tcservice.ITeamcity; -import org.jetbrains.annotations.Nullable; /** * Process pull request/untracked branch chain at particular server. */ public class PrChainsProcessor { + private static class Action { + public static final String HISTORY = "History"; + public static final String LATEST = "Latest"; + public static final String CHAIN = "Chain"; + } + /** Build chain processor. */ @Inject private BuildChainProcessor buildChainProcessor; @@ -81,7 +85,7 @@ public class PrChainsProcessor { */ @AutoProfiling public DsSummaryUi getTestFailuresSummary( - ITcBotUserCreds creds, + ICredentialsProv creds, String srvCode, String suiteId, String branchForTc, @@ -103,11 +107,11 @@ public DsSummaryUi getTestFailuresSummary( res.setJavaFlags(gitHubConnIgnited.config(), jiraIntegration.config()); LatestRebuildMode rebuild; - if (FullQueryParams.HISTORY.equals(act)) + if (Action.HISTORY.equals(act)) rebuild = LatestRebuildMode.ALL; - else if (FullQueryParams.LATEST.equals(act)) + else if (Action.LATEST.equals(act)) rebuild = LatestRebuildMode.LATEST; - else if (FullQueryParams.CHAIN.equals(act)) + else if (Action.CHAIN.equals(act)) rebuild = LatestRebuildMode.NONE; else rebuild = LatestRebuildMode.LATEST; @@ -212,13 +216,13 @@ public void initJiraAndGitInfo(DsChainUi chainStatus, @Nullable public List getBlockersSuitesStatuses(String buildTypeId, String branchForTc, String srvId, - ITcBotUserCreds prov) { + ICredentialsProv prov) { return getBlockersSuitesStatuses(buildTypeId, branchForTc, srvId, prov, SyncMode.RELOAD_QUEUED); } @Nullable public List getBlockersSuitesStatuses(String buildTypeId, String branchForTc, String srvId, - ITcBotUserCreds prov, SyncMode syncMode) { + ICredentialsProv prov, SyncMode syncMode) { //using here non persistent TC allows to skip update statistic ITeamcityIgnited tcIgnited = tcIgnitedProvider.server(srvId, prov); @@ -254,19 +258,19 @@ private List findBlockerFailures(FullChainRunCtx fullChainRunCtx, ITe .failedChildSuites() .map((ctx) -> { String normalizedBaseBranch = RunHistSync.normalizeBranch(baseBranch); - IRunHistory statInBaseBranch = tcIgnited.getSuiteRunHist(ctx.suiteId(), normalizedBaseBranch); + Integer baseBranchId = compactor.getStringIdIfPresent(normalizedBaseBranch); + IRunHistory statInBaseBranch = ctx.history(tcIgnited, baseBranchId); String suiteComment = ctx.getPossibleBlockerComment(compactor, statInBaseBranch, tcIgnited.config()); List failures = ctx.getFailedTests().stream().map(occurrence -> { - IRunHistory stat = tcIgnited.getTestRunHist(occurrence.getName(), normalizedBaseBranch); - + IRunHistory stat = occurrence.history(tcIgnited, baseBranchId); String testBlockerComment = TestCompactedMult.getPossibleBlockerComment(stat); if (!Strings.isNullOrEmpty(testBlockerComment)) { final DsTestFailureUi failure = new DsTestFailureUi(); - failure.initFromOccurrence(occurrence, tcIgnited, ctx.projectId(), ctx.branchName(), baseBranch); + failure.initFromOccurrence(occurrence, tcIgnited, ctx.projectId(), ctx.branchName(), baseBranch, baseBranchId); return failure; } diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java index bda0b0272..a0b40af5f 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java @@ -22,18 +22,17 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.inject.Inject; - import org.apache.ignite.tcbot.common.conf.ITcServerConfig; +import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; import org.apache.ignite.tcbot.engine.chain.BuildChainProcessor; import org.apache.ignite.tcbot.engine.chain.FullChainRunCtx; import org.apache.ignite.tcbot.engine.chain.LatestRebuildMode; import org.apache.ignite.tcbot.engine.chain.ProcessLogsMode; -import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; import org.apache.ignite.tcbot.engine.conf.ITcBotConfig; import org.apache.ignite.tcbot.engine.conf.ITrackedBranch; import org.apache.ignite.tcbot.engine.ui.DsChainUi; -import org.apache.ignite.tcbot.engine.ui.LrTestsFullSummaryUi; import org.apache.ignite.tcbot.engine.ui.DsSummaryUi; +import org.apache.ignite.tcbot.engine.ui.LrTestsFullSummaryUi; import org.apache.ignite.tcbot.persistence.IStringCompactor; import org.apache.ignite.tcignited.ITeamcityIgnited; import org.apache.ignite.tcignited.ITeamcityIgnitedProvider; @@ -107,7 +106,7 @@ public DsSummaryUi getTrackedBranchTestFailures( boolean includeScheduled = buildResMergeCnt == 1; final FullChainRunCtx ctx = chainProc.loadFullChainContext( - tcIgnited, + tcIgnited, chains, rebuild, logs, diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/BuildTimeRecordUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/BuildTimeRecordUi.java new file mode 100644 index 000000000..f411d8310 --- /dev/null +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/BuildTimeRecordUi.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.tcbot.engine.ui; + +@SuppressWarnings({"WeakerAccess", "PublicField"}) +public class BuildTimeRecordUi { + public String buildType; + public String averageDuration; + public String totalDuration; +} diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/BuildTimeResultUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/BuildTimeResultUi.java new file mode 100644 index 000000000..295a141af --- /dev/null +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/BuildTimeResultUi.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.tcbot.engine.ui; + +import org.apache.ignite.tcignited.buildtime.BuildTimeResult; + +import java.util.ArrayList; +import java.util.List; + +@SuppressWarnings({"WeakerAccess", "PublicField"}) +public class BuildTimeResultUi { + public List byBuildType = new ArrayList<>(); + public List timedOutByBuildType = new ArrayList<>(); +} diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java index 913c08a73..d8c87465e 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java @@ -24,13 +24,12 @@ import java.util.Objects; import java.util.stream.Stream; import javax.annotation.Nullable; - +import org.apache.ignite.internal.util.typedef.T2; +import org.apache.ignite.tcbot.common.util.CollectionUtil; import org.apache.ignite.tcbot.common.util.UrlUtil; import org.apache.ignite.tcbot.engine.chain.FullChainRunCtx; -import org.apache.ignite.tcbot.engine.chain.IMultTestOccurrence; import org.apache.ignite.tcbot.engine.chain.MultBuildRunCtx; -import org.apache.ignite.tcbot.common.util.CollectionUtil; -import org.apache.ignite.internal.util.typedef.T2; +import org.apache.ignite.tcbot.engine.chain.TestCompactedMult; import org.apache.ignite.tcbot.persistence.IStringCompactor; import org.apache.ignite.tcignited.ITeamcityIgnited; import org.apache.ignite.tcservice.model.conf.BuildType; @@ -208,18 +207,18 @@ public void initFromContext(ITeamcityIgnited tcIgnited, webToHist = buildWebLink(tcIgnited, ctx); webToBuild = buildWebLinkToBuild(tcIgnited, ctx); - Stream> allLongRunning = ctx.suites().flatMap( + Stream> allLongRunning = ctx.suites().flatMap( suite -> suite.getTopLongRunning().map(t -> new T2<>(suite, t)) ); - Comparator> durationComp + Comparator> durationComp = Comparator.comparing((pair) -> pair.get2().getAvgDurationMs()); CollectionUtil.top(allLongRunning, 3, durationComp).forEach( pairCtxAndOccur -> { MultBuildRunCtx suite = pairCtxAndOccur.get1(); - IMultTestOccurrence longRunningOccur = pairCtxAndOccur.get2(); + TestCompactedMult longRunningOccur = pairCtxAndOccur.get2(); - DsTestFailureUi failure = createOrrucForLongRun(tcIgnited, suite, longRunningOccur, baseBranchTc); + DsTestFailureUi failure = createOrrucForLongRun(tcIgnited, compactor, suite, longRunningOccur, baseBranchTc); failure.testName = "[" + suite.suiteName() + "] " + failure.testName; //may be separate field diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsHistoryStatUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsHistoryStatUi.java index a718e47be..6107c70a1 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsHistoryStatUi.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsHistoryStatUi.java @@ -23,7 +23,7 @@ /** * Detailed status of failures: Test/suite failure summary: contains statistic of failures and total runs for suite or for test. */ -public class DsHistoryStatUi { +@SuppressWarnings("PublicField") public class DsHistoryStatUi { /** Registered number of failures from TC helper DB */ @Nullable public Integer failures; diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java index 23fe3bdcb..3c98b34ff 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java @@ -21,6 +21,7 @@ * Detailed status of failures: Reference to some Issue with current suite or test detected by the Bot. * Currently contains only display name. */ +@SuppressWarnings({"WeakerAccess", "PublicField"}) public class DsProblemRef { public String name; public String webUrl; diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java index 8d25cf23a..eea92129e 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java @@ -29,19 +29,18 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; - import org.apache.ignite.tcbot.common.util.UrlUtil; -import org.apache.ignite.tcbot.engine.chain.IMultTestOccurrence; import org.apache.ignite.tcbot.engine.chain.MultBuildRunCtx; +import org.apache.ignite.tcbot.engine.chain.TestCompactedMult; import org.apache.ignite.tcbot.engine.issue.EventTemplates; import org.apache.ignite.tcbot.engine.ui.BotUrls.GetBuildLog; -import org.apache.ignite.tcignited.buildlog.ITestLogCheckResult; -import org.apache.ignite.tcignited.history.IRunHistory; import org.apache.ignite.tcbot.persistence.IStringCompactor; import org.apache.ignite.tcignited.ITeamcityIgnited; +import org.apache.ignite.tcignited.buildlog.ITestLogCheckResult; +import org.apache.ignite.tcignited.history.IRunHistory; -import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch; import static org.apache.ignite.tcbot.common.util.TimeUtil.millisToDurationPrintable; +import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch; /** @@ -164,9 +163,12 @@ public DsSuiteUi initFromContext(ITeamcityIgnited tcIgnited, name = suite.suiteName(); String failRateNormalizedBranch = normalizeBranch(baseBranch); + Integer baseBranchId = compactor.getStringIdIfPresent(failRateNormalizedBranch); + String curBranchNormalized = normalizeBranch(suite.branchName()); + Integer curBranchId = compactor.getStringIdIfPresent(curBranchNormalized); - IRunHistory baseBranchHist = initSuiteStat(tcIgnited, failRateNormalizedBranch, curBranchNormalized, suite.suiteId()); + IRunHistory baseBranchHist = initSuiteStat(tcIgnited, baseBranchId, curBranchId, suite); Set collect = suite.lastChangeUsers().collect(Collectors.toSet()); @@ -186,26 +188,28 @@ public DsSuiteUi initFromContext(ITeamcityIgnited tcIgnited, webToHistBaseBranch = buildWebLink(tcIgnited, suite, baseBranch); webToBuild = buildWebLinkToBuild(tcIgnited, suite); + Integer buildTypeIdId = suite.buildTypeIdId(); if (includeTests) { - List tests = suite.getFailedTests(); - Function function = foccur -> { - IRunHistory apply = tcIgnited.getTestRunHist(foccur.getName(), failRateNormalizedBranch); + List tests = suite.getFailedTests(); + Function function = testCompactedMult -> { + IRunHistory res = testCompactedMult.history(tcIgnited, baseBranchId); - return apply == null ? 0f : apply.getFailRate(); + return res == null ? 0f : res.getFailRate(); }; tests.sort(Comparator.comparing(function).reversed()); tests.forEach(occurrence -> { final DsTestFailureUi failure = new DsTestFailureUi(); - failure.initFromOccurrence(occurrence, tcIgnited, suite.projectId(), suite.branchName(), baseBranch); - failure.initStat(tcIgnited, failRateNormalizedBranch, curBranchNormalized); + failure.initFromOccurrence(occurrence, tcIgnited, suite.projectId(), + suite.branchName(), baseBranch, baseBranchId); + failure.initStat(occurrence, buildTypeIdId, tcIgnited, baseBranchId, curBranchId); testFailures.add(failure); }); suite.getTopLongRunning().forEach(occurrence -> { - final DsTestFailureUi failure = createOrrucForLongRun(tcIgnited, suite, occurrence, baseBranch); + final DsTestFailureUi failure = createOrrucForLongRun(tcIgnited, compactor, suite, occurrence, baseBranch); topLongRunning.add(failure); }); @@ -260,13 +264,10 @@ public DsSuiteUi initFromContext(ITeamcityIgnited tcIgnited, } private IRunHistory initSuiteStat(ITeamcityIgnited tcIgnited, - String failRateNormalizedBranch, - String curBranchNormalized, - String suiteId) { - if (Strings.isNullOrEmpty(suiteId)) - return null; - - final IRunHistory statInBaseBranch = tcIgnited.getSuiteRunHist(suiteId, failRateNormalizedBranch); + Integer failRateNormalizedBranch, + Integer curBranchNormalized, + MultBuildRunCtx suite) { + IRunHistory statInBaseBranch = suite.history(tcIgnited, failRateNormalizedBranch); if (statInBaseBranch != null) { failures = statInBaseBranch.getFailuresCount(); @@ -285,9 +286,8 @@ private IRunHistory initSuiteStat(ITeamcityIgnited tcIgnited, } IRunHistory latestRunsSrc = null; - if (!failRateNormalizedBranch.equals(curBranchNormalized)) { - - final IRunHistory statForStripe = tcIgnited.getSuiteRunHist(suiteId, curBranchNormalized); + if (!Objects.equals(failRateNormalizedBranch, curBranchNormalized)) { + IRunHistory statForStripe = suite.history(tcIgnited, curBranchNormalized); latestRunsSrc = statForStripe; latestRuns = statForStripe != null ? statForStripe.getLatestRunResults() : null; @@ -315,16 +315,18 @@ public static DsTestFailureUi createOccurForLogConsumer(Map.Entry } @Nonnull public static DsTestFailureUi createOrrucForLongRun(ITeamcityIgnited tcIgnited, - @Nonnull MultBuildRunCtx suite, - final IMultTestOccurrence occurrence, - @Nullable final String failRateBranch) { + IStringCompactor compactor, @Nonnull MultBuildRunCtx suite, + final TestCompactedMult occurrence, + @Nullable final String failRateBranch) { final DsTestFailureUi failure = new DsTestFailureUi(); - failure.initFromOccurrence(occurrence, tcIgnited, suite.projectId(), suite.branchName(), failRateBranch); + Integer baseBranchId = compactor.getStringIdIfPresent(normalizeBranch(failRateBranch)); + Integer buildTypeIdId = suite.buildTypeIdId(); + failure.initFromOccurrence(occurrence, tcIgnited, suite.projectId(), suite.branchName(), + failRateBranch, baseBranchId); - failure.initStat(tcIgnited, - normalizeBranch(failRateBranch), - normalizeBranch(suite.branchName())); + failure.initStat(occurrence, buildTypeIdId, tcIgnited, baseBranchId, + compactor.getStringIdIfPresent(normalizeBranch(suite.branchName()))); return failure; } diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java index b0cb5d9cf..6f9f12788 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java @@ -25,17 +25,15 @@ import java.util.regex.Pattern; import javax.annotation.Nonnull; import javax.annotation.Nullable; - import org.apache.ignite.tcbot.common.util.UrlUtil; -import org.apache.ignite.tcbot.engine.chain.IMultTestOccurrence; import org.apache.ignite.tcbot.engine.chain.TestCompactedMult; import org.apache.ignite.tcbot.engine.issue.EventTemplates; +import org.apache.ignite.tcignited.ITeamcityIgnited; import org.apache.ignite.tcignited.buildlog.LogMsgToWarn; import org.apache.ignite.tcignited.history.IRunHistory; -import org.apache.ignite.tcignited.ITeamcityIgnited; -import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch; import static org.apache.ignite.tcbot.common.util.TimeUtil.millisToDurationPrintable; +import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch; /** @@ -94,14 +92,16 @@ public class DsTestFailureUi { * @param failure test ocurrence (probably multiple) * @param tcIgn Teamcity. * @param projectId project ID. - * @param branchName - * @param baseBranchName base branch name (e.g. master). + * @param branchName current branch name. + * @param baseBranchName base branch name (e.g. master), without normalization. + * @param baseBranchId Normalized base branch ID (from compactor). */ - public void initFromOccurrence(@Nonnull final IMultTestOccurrence failure, + public void initFromOccurrence(@Nonnull final TestCompactedMult failure, @Nonnull final ITeamcityIgnited tcIgn, @Nullable final String projectId, @Nullable final String branchName, - @Nullable final String baseBranchName) { + @Nullable final String baseBranchName, + Integer baseBranchId) { name = failure.getName(); investigated = failure.isInvestigated(); curFailures = failure.failuresCount(); @@ -144,8 +144,7 @@ public void initFromOccurrence(@Nonnull final IMultTestOccurrence failure, webUrlBaseBranch = buildWebLink(tcIgn, full.test.id, projectId, baseBranchName); }); - final IRunHistory stat = tcIgn.getTestRunHist(name, normalizeBranch(baseBranchName)); - + final IRunHistory stat = failure.history(tcIgn, baseBranchId); blockerComment = TestCompactedMult.getPossibleBlockerComment(stat); } @@ -191,22 +190,22 @@ private static String buildWebLink(ITeamcityIgnited tcIgn, String id, } /** + * @param occurrence + * @param buildTypeIdId * @param tcIgnited TC service as Run stat supplier. - * @param failRateNormalizedBranch Base branch: Fail rate and flakyness detection normalized branch. + * @param baseBranchId Base branch: Fail rate and flakyness detection normalized branch. * @param curBranchNormalized Cur branch normalized. */ - public void initStat(ITeamcityIgnited tcIgnited, - String failRateNormalizedBranch, - String curBranchNormalized) { - - final IRunHistory stat = tcIgnited.getTestRunHist(name, failRateNormalizedBranch); - + public void initStat(TestCompactedMult occurrence, Integer buildTypeIdId, ITeamcityIgnited tcIgnited, + @Nullable Integer baseBranchId, + @Nullable Integer curBranchNormalized) { + final IRunHistory stat = occurrence.history(tcIgnited, baseBranchId); histBaseBranch.init(stat); - IRunHistory statForProblemsDetection = null; + IRunHistory statForProblemsDetection; - if (!curBranchNormalized.equals(failRateNormalizedBranch)) { - statForProblemsDetection = tcIgnited.getTestRunHist(name, curBranchNormalized); + if (!Objects.equals(curBranchNormalized, baseBranchId)) { + statForProblemsDetection = occurrence.history(tcIgnited, curBranchNormalized); if (statForProblemsDetection != null) { histCurBranch = new DsTestHistoryUi(); diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestHistoryUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestHistoryUi.java index 8b8597fb7..e0fe975d8 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestHistoryUi.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestHistoryUi.java @@ -27,7 +27,7 @@ */ public class DsTestHistoryUi { /** 'All the time' runs history statistic. */ - public DsHistoryStatUi allTime = new DsHistoryStatUi(); + @Deprecated public DsHistoryStatUi allTime = new DsHistoryStatUi(); /** Latest runs history statistic. */ public DsHistoryStatUi recent = new DsHistoryStatUi(); diff --git a/tcbot-github-ignited/README.md b/tcbot-github-ignited/README.md new file mode 100644 index 000000000..e25096c56 --- /dev/null +++ b/tcbot-github-ignited/README.md @@ -0,0 +1,3 @@ +TC Bot GitHub Ignited module +-------------------------------- +Module for requests to Ingnite caches containing GitHub data \ No newline at end of file diff --git a/tcbot-github-ignited/build.gradle b/tcbot-github-ignited/build.gradle new file mode 100644 index 000000000..e780b09f7 --- /dev/null +++ b/tcbot-github-ignited/build.gradle @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +apply plugin: 'java' + +dependencies { + compile (project(":tcbot-github")); + compile (project(":tcbot-persistence")); +} + \ No newline at end of file diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java b/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubConnIgnitedImpl.java similarity index 96% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java rename to tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubConnIgnitedImpl.java index 3415a832c..21214ef20 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java +++ b/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubConnIgnitedImpl.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ignite.ci.github.ignited; +package org.apache.ignite.githubignited; import java.util.HashSet; import java.util.List; @@ -25,21 +25,21 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.inject.Inject; import javax.inject.Provider; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; -import org.apache.ignite.ci.db.TcHelperDb; +import org.apache.ignite.githubservice.IGitHubConnection; import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; import org.apache.ignite.tcbot.common.interceptor.MonitoredTask; +import org.apache.ignite.tcbot.persistence.CacheConfigs; import org.apache.ignite.tcbot.persistence.scheduler.IScheduler; import org.apache.ignite.ci.github.GitHubBranchKey; import org.apache.ignite.ci.github.GitHubBranchShort; import org.apache.ignite.ci.github.PullRequest; -import org.apache.ignite.ci.github.pure.IGitHubConnection; import org.apache.ignite.tcbot.common.conf.IGitHubConfig; -import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -86,8 +86,8 @@ public void init(IGitHubConnection conn) { srvIdMaskHigh = Math.abs(srvCode.hashCode()); Ignite ignite = igniteProvider.get(); - prCache = ignite.getOrCreateCache(TcHelperDb.getCache8PartsConfig(GIT_HUB_PR)); - branchCache = ignite.getOrCreateCache(TcHelperDb.getCache8PartsConfig(GIT_HUB_BRANCHES)); + prCache = ignite.getOrCreateCache(CacheConfigs.getCache8PartsConfig(GIT_HUB_PR)); + branchCache = ignite.getOrCreateCache(CacheConfigs.getCache8PartsConfig(GIT_HUB_BRANCHES)); } /** {@inheritDoc} */ @@ -148,7 +148,7 @@ private void actualizeBranches() { * @param taskName Task name. * @return Task name concatenated with server name. */ - @NotNull + @Nonnull private String taskName(String taskName) { return IGitHubConnIgnited.class.getSimpleName() + "." + taskName + "." + srvCode; } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubIgnitedModule.java b/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubIgnitedModule.java similarity index 92% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubIgnitedModule.java rename to tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubIgnitedModule.java index 22c39f918..9c199651b 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubIgnitedModule.java +++ b/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubIgnitedModule.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ignite.ci.github.ignited; +package org.apache.ignite.githubignited; import com.google.inject.AbstractModule; import com.google.inject.internal.SingletonScope; -import org.apache.ignite.ci.github.pure.GitHubIntegrationModule; +import org.apache.ignite.githubservice.GitHubIntegrationModule; /** * Requires {@link org.apache.ignite.ci.di.scheduler.SchedulerModule} to be installed diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubIgnitedProvImpl.java b/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubIgnitedProvImpl.java similarity index 92% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubIgnitedProvImpl.java rename to tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubIgnitedProvImpl.java index cf8fbb02d..99e8a2d9a 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubIgnitedProvImpl.java +++ b/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubIgnitedProvImpl.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ignite.ci.github.ignited; +package org.apache.ignite.githubignited; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -24,8 +24,8 @@ import java.util.concurrent.TimeUnit; import javax.inject.Inject; import javax.inject.Provider; -import org.apache.ignite.ci.github.pure.IGitHubConnection; -import org.apache.ignite.ci.github.pure.IGitHubConnectionProvider; +import org.apache.ignite.githubservice.IGitHubConnection; +import org.apache.ignite.githubservice.IGitHubConnectionProvider; import org.apache.ignite.tcbot.common.exeption.ExceptionUtil; class GitHubIgnitedProvImpl implements IGitHubConnIgnitedProvider { diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/IGitHubConnIgnited.java b/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/IGitHubConnIgnited.java similarity index 97% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/IGitHubConnIgnited.java rename to tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/IGitHubConnIgnited.java index 125cc37ce..1ceb3226d 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/IGitHubConnIgnited.java +++ b/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/IGitHubConnIgnited.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ignite.ci.github.ignited; +package org.apache.ignite.githubignited; import java.util.List; import org.apache.ignite.ci.github.PullRequest; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/IGitHubConnIgnitedProvider.java b/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/IGitHubConnIgnitedProvider.java similarity index 95% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/IGitHubConnIgnitedProvider.java rename to tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/IGitHubConnIgnitedProvider.java index 729e77e81..e1a94a8c0 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/IGitHubConnIgnitedProvider.java +++ b/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/IGitHubConnIgnitedProvider.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ignite.ci.github.ignited; +package org.apache.ignite.githubignited; /** * diff --git a/tcbot-github/README.md b/tcbot-github/README.md new file mode 100644 index 000000000..c1322e463 --- /dev/null +++ b/tcbot-github/README.md @@ -0,0 +1,3 @@ +TC Bot GitHub (service) module +-------------------------------- +Module for pure (non cached) requests to GitHub service \ No newline at end of file diff --git a/tcbot-github/build.gradle b/tcbot-github/build.gradle new file mode 100644 index 000000000..2bacd4086 --- /dev/null +++ b/tcbot-github/build.gradle @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +apply plugin: 'java' + +dependencies { + compile (project(":tcbot-common")); + // GitHub integration shares entries for pure and Ignited connection, so persistence module is here for interfaces/annotations + compile (project(":tcbot-persistence")); + + testCompile group: 'junit', name: 'junit', version: junitVer + testCompile group: 'org.mockito', name: 'mockito-core', version: mockitoVer +} + + +processResources { + from(sourceSets.test.resources.srcDirs) { + include "**/*.json" + } +} + +test { + // set JVM arguments for the test JVM(s) + jvmArgs "-XX:+IgnoreUnrecognizedVMOptions", + "--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED", + "--add-exports=java.base/sun.nio.ch=ALL-UNNAMED", + "--add-exports=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED", + "--add-exports=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED", + "--add-exports=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED", + "--illegal-access=permit" +} diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/GitHubBranch.java b/tcbot-github/src/main/java/org/apache/ignite/ci/github/GitHubBranch.java similarity index 99% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/GitHubBranch.java rename to tcbot-github/src/main/java/org/apache/ignite/ci/github/GitHubBranch.java index 804fe6545..9359342f7 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/GitHubBranch.java +++ b/tcbot-github/src/main/java/org/apache/ignite/ci/github/GitHubBranch.java @@ -16,9 +16,10 @@ */ package org.apache.ignite.ci.github; -import java.util.Objects; import org.apache.ignite.tcbot.persistence.Persisted; +import java.util.Objects; + @Persisted public class GitHubBranch { /** Label. */ diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/GitHubBranchKey.java b/tcbot-github/src/main/java/org/apache/ignite/ci/github/GitHubBranchKey.java similarity index 95% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/GitHubBranchKey.java rename to tcbot-github/src/main/java/org/apache/ignite/ci/github/GitHubBranchKey.java index e3abf77c7..976bf639f 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/GitHubBranchKey.java +++ b/tcbot-github/src/main/java/org/apache/ignite/ci/github/GitHubBranchKey.java @@ -17,9 +17,10 @@ package org.apache.ignite.ci.github; import com.google.common.base.Strings; -import java.util.Objects; import org.apache.ignite.tcbot.persistence.Persisted; -import org.jetbrains.annotations.NotNull; + +import javax.annotation.Nonnull; +import java.util.Objects; @Persisted public class GitHubBranchKey implements Comparable { @@ -63,7 +64,7 @@ public String branchName() { } /** {@inheritDoc} */ - @Override public int compareTo(@NotNull GitHubBranchKey o) { + @Override public int compareTo(@Nonnull GitHubBranchKey o) { int compare = Integer.compare(srvId, o.srvId()); if (compare != 0) return compare; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/GitHubBranchShort.java b/tcbot-github/src/main/java/org/apache/ignite/ci/github/GitHubBranchShort.java similarity index 99% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/GitHubBranchShort.java rename to tcbot-github/src/main/java/org/apache/ignite/ci/github/GitHubBranchShort.java index 496b4ceb9..bf3a31d97 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/GitHubBranchShort.java +++ b/tcbot-github/src/main/java/org/apache/ignite/ci/github/GitHubBranchShort.java @@ -16,9 +16,10 @@ */ package org.apache.ignite.ci.github; -import java.util.Objects; import org.apache.ignite.tcbot.persistence.Persisted; +import java.util.Objects; + /** * */ diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/GitHubCommit.java b/tcbot-github/src/main/java/org/apache/ignite/ci/github/GitHubCommit.java similarity index 99% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/GitHubCommit.java rename to tcbot-github/src/main/java/org/apache/ignite/ci/github/GitHubCommit.java index 7cad8cf9a..3d1498340 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/GitHubCommit.java +++ b/tcbot-github/src/main/java/org/apache/ignite/ci/github/GitHubCommit.java @@ -16,9 +16,10 @@ */ package org.apache.ignite.ci.github; -import java.util.Objects; import org.apache.ignite.tcbot.persistence.Persisted; +import java.util.Objects; + @Persisted public class GitHubCommit { /** Sha of the commit. */ diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/GitHubUser.java b/tcbot-github/src/main/java/org/apache/ignite/ci/github/GitHubUser.java similarity index 99% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/GitHubUser.java rename to tcbot-github/src/main/java/org/apache/ignite/ci/github/GitHubUser.java index 767dd8fbe..71510e06b 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/GitHubUser.java +++ b/tcbot-github/src/main/java/org/apache/ignite/ci/github/GitHubUser.java @@ -17,9 +17,10 @@ package org.apache.ignite.ci.github; import com.google.gson.annotations.SerializedName; -import java.util.Objects; import org.apache.ignite.tcbot.persistence.Persisted; +import java.util.Objects; + @Persisted public class GitHubUser { @SerializedName("login") private String login; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/PullRequest.java b/tcbot-github/src/main/java/org/apache/ignite/ci/github/PullRequest.java similarity index 98% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/PullRequest.java rename to tcbot-github/src/main/java/org/apache/ignite/ci/github/PullRequest.java index 24a1364c9..84b338b06 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/PullRequest.java +++ b/tcbot-github/src/main/java/org/apache/ignite/ci/github/PullRequest.java @@ -24,7 +24,8 @@ import org.apache.ignite.tcbot.persistence.IVersionedEntity; import org.apache.ignite.tcbot.persistence.Persisted; -import org.jetbrains.annotations.Nullable; + +import javax.annotation.Nullable; /** * @@ -167,7 +168,7 @@ public GitHubBranch head() { return LATEST_VERSION; } - @Nullable public String lastCommitShaShort() { + @Nullable public String lastCommitShaShort() { String sha = head().sha(); return Strings.isNullOrEmpty(sha) ? null : sha.substring(0, INCLUDE_SHORT_VER); diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/pure/GitHubCachingProvider.java b/tcbot-github/src/main/java/org/apache/ignite/githubservice/GitHubCachingProvider.java similarity index 97% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/pure/GitHubCachingProvider.java rename to tcbot-github/src/main/java/org/apache/ignite/githubservice/GitHubCachingProvider.java index ea73be293..d35244225 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/pure/GitHubCachingProvider.java +++ b/tcbot-github/src/main/java/org/apache/ignite/githubservice/GitHubCachingProvider.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ignite.ci.github.pure; +package org.apache.ignite.githubservice; import com.google.common.base.Preconditions; import com.google.common.base.Strings; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/pure/GitHubConnectionImpl.java b/tcbot-github/src/main/java/org/apache/ignite/githubservice/GitHubConnectionImpl.java similarity index 96% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/pure/GitHubConnectionImpl.java rename to tcbot-github/src/main/java/org/apache/ignite/githubservice/GitHubConnectionImpl.java index 8d5a41e09..832cf2f8a 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/pure/GitHubConnectionImpl.java +++ b/tcbot-github/src/main/java/org/apache/ignite/githubservice/GitHubConnectionImpl.java @@ -14,24 +14,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ignite.ci.github.pure; +package org.apache.ignite.githubservice; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; +import org.apache.ignite.tcbot.common.conf.IDataSourcesConfigSupplier; import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; import org.apache.ignite.ci.github.GitHubBranchShort; import org.apache.ignite.ci.github.PullRequest; import org.apache.ignite.tcbot.common.conf.IGitHubConfig; -import org.apache.ignite.tcbot.engine.conf.ITcBotConfig; import org.apache.ignite.tcbot.common.util.HttpUtil; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import javax.inject.Inject; import java.io.IOException; import java.io.InputStream; @@ -54,7 +53,7 @@ class GitHubConnectionImpl implements IGitHubConnection { /** Config. */ @Inject - private ITcBotConfig cfg; + private IDataSourcesConfigSupplier cfg; /** Service (server) code. */ private String srvCode; @@ -152,7 +151,7 @@ class GitHubConnectionImpl implements IGitHubConnection { return readOnePage(outLinkNext, url, rspHeaders, tok); } - @NotNull public String getApiUrlMandatory() { + @Nonnull public String getApiUrlMandatory() { String gitApiUrl = config().gitApiUrl(); Preconditions.checkState(!isNullOrEmpty(gitApiUrl), "Git API URL is not configured for this server."); diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/pure/GitHubIntegrationModule.java b/tcbot-github/src/main/java/org/apache/ignite/githubservice/GitHubIntegrationModule.java similarity index 96% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/pure/GitHubIntegrationModule.java rename to tcbot-github/src/main/java/org/apache/ignite/githubservice/GitHubIntegrationModule.java index 8b2911ef0..63090ad53 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/pure/GitHubIntegrationModule.java +++ b/tcbot-github/src/main/java/org/apache/ignite/githubservice/GitHubIntegrationModule.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ignite.ci.github.pure; +package org.apache.ignite.githubservice; import com.google.inject.AbstractModule; import com.google.inject.internal.SingletonScope; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/pure/IGitHubConnection.java b/tcbot-github/src/main/java/org/apache/ignite/githubservice/IGitHubConnection.java similarity index 97% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/pure/IGitHubConnection.java rename to tcbot-github/src/main/java/org/apache/ignite/githubservice/IGitHubConnection.java index 8f8f420f4..e806f3728 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/pure/IGitHubConnection.java +++ b/tcbot-github/src/main/java/org/apache/ignite/githubservice/IGitHubConnection.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ignite.ci.github.pure; +package org.apache.ignite.githubservice; import com.google.common.base.Strings; import java.util.List; @@ -23,9 +23,9 @@ import org.apache.ignite.ci.github.GitHubBranchShort; import org.apache.ignite.ci.github.PullRequest; import org.apache.ignite.tcbot.common.conf.IGitHubConfig; -import org.jetbrains.annotations.Nullable; import javax.annotation.Nonnull; +import javax.annotation.Nullable; /** * GitHub pure connection diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/pure/IGitHubConnectionProvider.java b/tcbot-github/src/main/java/org/apache/ignite/githubservice/IGitHubConnectionProvider.java similarity index 95% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/pure/IGitHubConnectionProvider.java rename to tcbot-github/src/main/java/org/apache/ignite/githubservice/IGitHubConnectionProvider.java index 3da468f3e..e089fc109 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/pure/IGitHubConnectionProvider.java +++ b/tcbot-github/src/main/java/org/apache/ignite/githubservice/IGitHubConnectionProvider.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ignite.ci.github.pure; +package org.apache.ignite.githubservice; public interface IGitHubConnectionProvider { /** diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/github/pure/GitHubPrsParseTest.java b/tcbot-github/src/test/java/org/apache/ignite/githubservice/GitHubPrsParseTest.java similarity index 98% rename from ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/github/pure/GitHubPrsParseTest.java rename to tcbot-github/src/test/java/org/apache/ignite/githubservice/GitHubPrsParseTest.java index 8a826579d..754fcc775 100644 --- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/github/pure/GitHubPrsParseTest.java +++ b/tcbot-github/src/test/java/org/apache/ignite/githubservice/GitHubPrsParseTest.java @@ -14,7 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ignite.ci.github.pure; + +package org.apache.ignite.githubservice; import com.google.common.base.Preconditions; import com.google.gson.Gson; diff --git a/ignite-tc-helper-web/src/test/resources/prsList.json b/tcbot-github/src/test/resources/prsList.json similarity index 100% rename from ignite-tc-helper-web/src/test/resources/prsList.json rename to tcbot-github/src/test/resources/prsList.json diff --git a/tcbot-jira-ignited/build.gradle b/tcbot-jira-ignited/build.gradle new file mode 100644 index 000000000..b8fed7fd9 --- /dev/null +++ b/tcbot-jira-ignited/build.gradle @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +apply plugin: 'java' + +dependencies { + compile (project(":tcbot-jira")); + compile (project(":tcbot-persistence")); +} + \ No newline at end of file diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java b/tcbot-jira-ignited/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java similarity index 92% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java rename to tcbot-jira-ignited/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java index 1f2a69903..c11c8b6c1 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java +++ b/tcbot-jira-ignited/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java @@ -18,18 +18,20 @@ package org.apache.ignite.ci.jira.ignited; import java.util.Objects; -import org.apache.ignite.ci.jira.pure.Fields; -import org.apache.ignite.ci.jira.pure.Status; -import org.apache.ignite.ci.jira.pure.Ticket; +import org.apache.ignite.jiraservice.Fields; +import org.apache.ignite.jiraservice.Status; +import org.apache.ignite.jiraservice.Ticket; import org.apache.ignite.ci.tcbot.common.StringFieldCompacted; import org.apache.ignite.tcbot.persistence.IStringCompactor; -import org.jetbrains.annotations.Nullable; +import org.apache.ignite.tcbot.persistence.Persisted; + +import javax.annotation.Nullable; /** * */ +@Persisted public class TicketCompacted { - public static final String PROJECT_DELIM = "-"; /** Id. */ public long id; @@ -70,7 +72,7 @@ public Ticket toTicket(IStringCompactor comp, String projectCode) { Ticket ticket = new Ticket(); ticket.id = id; - ticket.key = projectCode + PROJECT_DELIM + igniteId; + ticket.key = projectCode + Ticket.PROJECT_DELIM + igniteId; ticket.fields = new Fields(); ticket.fields.status = new Status(comp.getStringFromId(status)); ticket.fields.summary = summary != null ? summary.getValue() : null; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/IJiraIgnited.java b/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/IJiraIgnited.java similarity index 83% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/IJiraIgnited.java rename to tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/IJiraIgnited.java index 24a3086ef..4eda1921c 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/IJiraIgnited.java +++ b/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/IJiraIgnited.java @@ -14,13 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ignite.ci.jira.ignited; +package org.apache.ignite.jiraignited; import java.io.IOException; import java.util.Set; -import org.apache.ignite.ci.jira.pure.Ticket; +import org.apache.ignite.jiraservice.Ticket; import org.apache.ignite.tcbot.common.conf.IJiraServerConfig; +import javax.annotation.Nullable; + /** * */ @@ -52,4 +54,15 @@ public interface IJiraIgnited { public String postJiraComment(String ticket, String comment) throws IOException; public IJiraServerConfig config(); + + /** + * @param srvId Server id. + * @return integer representation of server ID. + */ + public static int serverIdToInt(@Nullable final String srvId) { + if (srvId == null) + return 0; + + return Math.abs(srvId.hashCode()); + } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/IJiraIgnitedProvider.java b/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/IJiraIgnitedProvider.java similarity index 95% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/IJiraIgnitedProvider.java rename to tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/IJiraIgnitedProvider.java index f1620a2b1..afaea3f26 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/IJiraIgnitedProvider.java +++ b/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/IJiraIgnitedProvider.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ignite.ci.jira.ignited; +package org.apache.ignite.jiraignited; /** * diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraIgnited.java b/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraIgnited.java similarity index 90% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraIgnited.java rename to tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraIgnited.java index 38190e1ad..1c7542449 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraIgnited.java +++ b/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraIgnited.java @@ -14,15 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ignite.ci.jira.ignited; +package org.apache.ignite.jiraignited; import java.io.IOException; import java.util.Set; import javax.inject.Inject; -import org.apache.ignite.ci.jira.pure.IJiraIntegration; -import org.apache.ignite.ci.jira.pure.Ticket; +import org.apache.ignite.jiraservice.IJiraIntegration; +import org.apache.ignite.jiraservice.Ticket; import org.apache.ignite.tcbot.common.conf.IJiraServerConfig; -import org.apache.ignite.tcignited.ITeamcityIgnited; /** * @@ -51,7 +50,7 @@ public void init(IJiraIntegration jira) { srvCode = jira.config().getCode(); - srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvCode); + srvIdMaskHigh = IJiraIgnited.serverIdToInt(srvCode); jiraTicketDao.init(); } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraIgnitedModule.java b/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraIgnitedModule.java similarity index 92% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraIgnitedModule.java rename to tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraIgnitedModule.java index 4126b5845..c58341fd7 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraIgnitedModule.java +++ b/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraIgnitedModule.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ignite.ci.jira.ignited; +package org.apache.ignite.jiraignited; import com.google.inject.AbstractModule; import com.google.inject.internal.SingletonScope; -import org.apache.ignite.ci.jira.pure.JiraIntegrationModule; +import org.apache.ignite.jiraservice.JiraIntegrationModule; /** * diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraIgnitedProvider.java b/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraIgnitedProvider.java similarity index 92% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraIgnitedProvider.java rename to tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraIgnitedProvider.java index 50852fe8f..4f4571b13 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraIgnitedProvider.java +++ b/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraIgnitedProvider.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ignite.ci.jira.ignited; +package org.apache.ignite.jiraignited; import com.google.common.base.Strings; import com.google.common.cache.Cache; @@ -23,8 +23,8 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import javax.inject.Inject; -import org.apache.ignite.ci.jira.pure.IJiraIntegration; -import org.apache.ignite.ci.jira.pure.IJiraIntegrationProvider; +import org.apache.ignite.jiraservice.IJiraIntegration; +import org.apache.ignite.jiraservice.IJiraIntegrationProvider; import org.apache.ignite.tcbot.common.exeption.ExceptionUtil; /** diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketDao.java b/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraTicketDao.java similarity index 93% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketDao.java rename to tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraTicketDao.java index 361860f68..5956b5a66 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketDao.java +++ b/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraTicketDao.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.ci.jira.ignited; +package org.apache.ignite.jiraignited; import com.google.common.base.Preconditions; import java.util.Collection; @@ -28,10 +28,11 @@ import javax.inject.Provider; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; -import org.apache.ignite.ci.db.TcHelperDb; +import org.apache.ignite.ci.jira.ignited.TicketCompacted; import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; import org.apache.ignite.tcbot.common.interceptor.GuavaCached; -import org.apache.ignite.ci.jira.pure.Ticket; +import org.apache.ignite.jiraservice.Ticket; +import org.apache.ignite.tcbot.persistence.CacheConfigs; import org.apache.ignite.tcbot.persistence.IStringCompactor; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.U; @@ -56,7 +57,7 @@ public class JiraTicketDao { * */ public void init() { - jiraCache = igniteProvider.get().getOrCreateCache(TcHelperDb.getCache8PartsConfig(TEAMCITY_JIRA_TICKET_CACHE_NAME)); + jiraCache = igniteProvider.get().getOrCreateCache(CacheConfigs.getCache8PartsConfig(TEAMCITY_JIRA_TICKET_CACHE_NAME)); } /** diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketSync.java b/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraTicketSync.java similarity index 91% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketSync.java rename to tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraTicketSync.java index 6664e876b..35eff37a5 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketSync.java +++ b/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraTicketSync.java @@ -15,25 +15,24 @@ * limitations under the License. */ -package org.apache.ignite.ci.jira.ignited; +package org.apache.ignite.jiraignited; import java.lang.reflect.Field; import java.util.Arrays; import java.util.Collection; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import javax.annotation.Nonnull; import javax.inject.Inject; import org.apache.ignite.tcbot.common.interceptor.MonitoredTask; import org.apache.ignite.tcbot.persistence.scheduler.IScheduler; -import org.apache.ignite.ci.jira.Tickets; -import org.apache.ignite.ci.jira.pure.Fields; -import org.apache.ignite.ci.jira.pure.IJiraIntegration; -import org.apache.ignite.ci.jira.pure.IJiraIntegrationProvider; -import org.apache.ignite.ci.jira.pure.Ticket; +import org.apache.ignite.jiraservice.Tickets; +import org.apache.ignite.jiraservice.Fields; +import org.apache.ignite.jiraservice.IJiraIntegration; +import org.apache.ignite.jiraservice.IJiraIntegrationProvider; +import org.apache.ignite.jiraservice.Ticket; import org.apache.ignite.tcbot.common.conf.IJiraServerConfig; -import org.apache.ignite.tcignited.ITeamcityIgnited; import org.apache.ignite.internal.util.typedef.F; -import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -83,7 +82,7 @@ public String incrementalUpdate(String srvCode) { * @param srvCode Service ID * @return Task name concatenated with server name. */ - @NotNull + @Nonnull private String taskName(String taskName, String srvCode) { return JiraTicketSync.class.getSimpleName() + "." + taskName + "." + srvCode; } @@ -92,9 +91,10 @@ private String taskName(String taskName, String srvCode) { * @param srvCode Server internal identification. * @param fullResync full or incremental. */ + @SuppressWarnings("WeakerAccess") @MonitoredTask(name = "Actualize Jira(srv, full resync)", nameExtArgsIndexes = {0, 1}) protected String actualizeJiraTickets(String srvCode, boolean fullResync) { - int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvCode); + int srvIdMaskHigh = IJiraIgnited.serverIdToInt(srvCode); IJiraIntegration jira = jiraIntegrationProvider.server(srvCode); String reqFields = Arrays.stream(Fields.class.getDeclaredFields()) diff --git a/tcbot-jira/README.md b/tcbot-jira/README.md new file mode 100644 index 000000000..241f97060 --- /dev/null +++ b/tcbot-jira/README.md @@ -0,0 +1,3 @@ +TC Bot JIRA (service) module +-------------------------------- +Module for pure (non cached) requests to JIRA service \ No newline at end of file diff --git a/tcbot-jira/build.gradle b/tcbot-jira/build.gradle new file mode 100644 index 000000000..ca6c2cc38 --- /dev/null +++ b/tcbot-jira/build.gradle @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +apply plugin: 'java' + +dependencies { + compile (project(":tcbot-common")); + // JIRA integration shares entries for pure and Ignited connection, so persistence module is here for interfaces/annotations + compile (project(":tcbot-persistence")); + + testCompile group: 'junit', name: 'junit', version: junitVer + testCompile group: 'org.mockito', name: 'mockito-core', version: mockitoVer +} + \ No newline at end of file diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Fields.java b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/Fields.java similarity index 97% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Fields.java rename to tcbot-jira/src/main/java/org/apache/ignite/jiraservice/Fields.java index 02ed0b229..914b2c4d4 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Fields.java +++ b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/Fields.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.ci.jira.pure; +package org.apache.ignite.jiraservice; import com.google.common.base.MoreObjects; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/IJiraIntegration.java b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/IJiraIntegration.java similarity index 96% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/IJiraIntegration.java rename to tcbot-jira/src/main/java/org/apache/ignite/jiraservice/IJiraIntegration.java index cfced3cf4..6d44b9da4 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/IJiraIntegration.java +++ b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/IJiraIntegration.java @@ -15,10 +15,9 @@ * limitations under the License. */ -package org.apache.ignite.ci.jira.pure; +package org.apache.ignite.jiraservice; import java.io.IOException; -import org.apache.ignite.ci.jira.Tickets; import org.apache.ignite.tcbot.common.conf.IJiraServerConfig; /** diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/IJiraIntegrationProvider.java b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/IJiraIntegrationProvider.java similarity index 95% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/IJiraIntegrationProvider.java rename to tcbot-jira/src/main/java/org/apache/ignite/jiraservice/IJiraIntegrationProvider.java index 984233b99..688651857 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/IJiraIntegrationProvider.java +++ b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/IJiraIntegrationProvider.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.ci.jira.pure; +package org.apache.ignite.jiraservice; /** * diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Jira.java b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/Jira.java similarity index 95% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Jira.java rename to tcbot-jira/src/main/java/org/apache/ignite/jiraservice/Jira.java index cdef64787..b2d494340 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Jira.java +++ b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/Jira.java @@ -15,14 +15,13 @@ * limitations under the License. */ -package org.apache.ignite.ci.jira.pure; +package org.apache.ignite.jiraservice; import com.google.common.base.Preconditions; import com.google.gson.Gson; +import org.apache.ignite.tcbot.common.conf.IDataSourcesConfigSupplier; import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; -import org.apache.ignite.ci.jira.Tickets; import org.apache.ignite.tcbot.common.conf.IJiraServerConfig; -import org.apache.ignite.tcbot.engine.conf.ITcBotConfig; import org.apache.ignite.tcbot.common.util.HttpUtil; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; @@ -44,7 +43,7 @@ class Jira implements IJiraIntegration { private String srvCode; /** Config. */ - @Inject ITcBotConfig cfg; + @Inject private IDataSourcesConfigSupplier cfg; /** {@inheritDoc} */ @Override public void init(String srvCode) { diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/JiraIntegrationModule.java b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/JiraIntegrationModule.java similarity index 96% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/JiraIntegrationModule.java rename to tcbot-jira/src/main/java/org/apache/ignite/jiraservice/JiraIntegrationModule.java index 3c19a9f75..325a94eed 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/JiraIntegrationModule.java +++ b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/JiraIntegrationModule.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.ci.jira.pure; +package org.apache.ignite.jiraservice; import com.google.inject.AbstractModule; import com.google.inject.internal.SingletonScope; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/JiraIntegrationProvider.java b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/JiraIntegrationProvider.java similarity index 98% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/JiraIntegrationProvider.java rename to tcbot-jira/src/main/java/org/apache/ignite/jiraservice/JiraIntegrationProvider.java index fcfd47306..62d648c3f 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/JiraIntegrationProvider.java +++ b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/JiraIntegrationProvider.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.ci.jira.pure; +package org.apache.ignite.jiraservice; import com.google.common.base.Strings; import com.google.common.cache.Cache; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Status.java b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/Status.java similarity index 97% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Status.java rename to tcbot-jira/src/main/java/org/apache/ignite/jiraservice/Status.java index c1ad37a95..3cc37e990 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Status.java +++ b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/Status.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.ci.jira.pure; +package org.apache.ignite.jiraservice; /** * Status for Jira ticket. diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Ticket.java b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/Ticket.java similarity index 93% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Ticket.java rename to tcbot-jira/src/main/java/org/apache/ignite/jiraservice/Ticket.java index 256bb3d85..2221c6ce6 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Ticket.java +++ b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/Ticket.java @@ -15,10 +15,9 @@ * limitations under the License. */ -package org.apache.ignite.ci.jira.pure; +package org.apache.ignite.jiraservice; import com.google.common.base.MoreObjects; -import org.apache.ignite.ci.jira.ignited.TicketCompacted; import javax.annotation.Nullable; @@ -26,6 +25,7 @@ * See example of GSON here https://issues.apache.org/jira/rest/api/2/issue/IGNITE-123 */ public class Ticket { + public static final String PROJECT_DELIM = "-"; /** Id. */ public long id; @@ -47,7 +47,7 @@ public Ticket(String ticketKey) { * @return Ignite ticket Number ignoring project code (like 123 in IGNITE-123). */ public int keyWithoutProject(String projectCode) { - String ticketPrefix = projectCode + TicketCompacted.PROJECT_DELIM; + String ticketPrefix = projectCode + PROJECT_DELIM; return Integer.valueOf(key.substring(ticketPrefix.length())); } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/Tickets.java b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/Tickets.java similarity index 95% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/Tickets.java rename to tcbot-jira/src/main/java/org/apache/ignite/jiraservice/Tickets.java index 4ce85c527..125baa6e9 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/Tickets.java +++ b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/Tickets.java @@ -15,11 +15,11 @@ * limitations under the License. */ -package org.apache.ignite.ci.jira; +package org.apache.ignite.jiraservice; import java.util.Collection; import java.util.Collections; -import org.apache.ignite.ci.jira.pure.Ticket; +import org.apache.ignite.jiraservice.Ticket; /** * See example of GSON here https://issues.apache.org/jira/rest/api/2/search?jql=project%20=%20IGNITE%20order%20by%20updated%20DESC&fields=status diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/package-info.java b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/package-info.java similarity index 96% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/package-info.java rename to tcbot-jira/src/main/java/org/apache/ignite/jiraservice/package-info.java index f7c4ecfb8..5699df1a2 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/package-info.java +++ b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/package-info.java @@ -19,4 +19,4 @@ * This package is intended for interacting with Jira servers. It contains classes which is needed for injection Jira * integration service instance via Guice. And interfaces and classes which provide specified functionality. */ -package org.apache.ignite.ci.jira; +package org.apache.ignite.jiraservice; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/NamedTask.java b/tcbot-persistence/src/main/java/org/apache/ignite/tcbot/persistence/scheduler/NamedTask.java similarity index 98% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/NamedTask.java rename to tcbot-persistence/src/main/java/org/apache/ignite/tcbot/persistence/scheduler/NamedTask.java index a5b2c37a9..8c1ef609b 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/NamedTask.java +++ b/tcbot-persistence/src/main/java/org/apache/ignite/tcbot/persistence/scheduler/NamedTask.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ignite.ci.di.scheduler; +package org.apache.ignite.tcbot.persistence.scheduler; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.StampedLock; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/SchedulerModule.java b/tcbot-persistence/src/main/java/org/apache/ignite/tcbot/persistence/scheduler/SchedulerModule.java similarity index 91% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/SchedulerModule.java rename to tcbot-persistence/src/main/java/org/apache/ignite/tcbot/persistence/scheduler/SchedulerModule.java index 81cd86e04..2cd2ea722 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/SchedulerModule.java +++ b/tcbot-persistence/src/main/java/org/apache/ignite/tcbot/persistence/scheduler/SchedulerModule.java @@ -15,11 +15,10 @@ * limitations under the License. */ -package org.apache.ignite.ci.di.scheduler; +package org.apache.ignite.tcbot.persistence.scheduler; import com.google.inject.AbstractModule; import com.google.inject.internal.SingletonScope; -import org.apache.ignite.tcbot.persistence.scheduler.IScheduler; public class SchedulerModule extends AbstractModule { /** {@inheritDoc} */ diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java b/tcbot-persistence/src/main/java/org/apache/ignite/tcbot/persistence/scheduler/TcBotScheduler.java similarity index 97% rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java rename to tcbot-persistence/src/main/java/org/apache/ignite/tcbot/persistence/scheduler/TcBotScheduler.java index 791ae2141..5728e0ec0 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java +++ b/tcbot-persistence/src/main/java/org/apache/ignite/tcbot/persistence/scheduler/TcBotScheduler.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ignite.ci.di.scheduler; +package org.apache.ignite.tcbot.persistence.scheduler; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -30,7 +30,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.apache.ignite.tcbot.common.interceptor.MonitoredTask; -import org.apache.ignite.tcbot.persistence.scheduler.IScheduler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tcbot-server-node/build.gradle b/tcbot-server-node/build.gradle index 3ffcf74ef..0bc43f859 100644 --- a/tcbot-server-node/build.gradle +++ b/tcbot-server-node/build.gradle @@ -18,9 +18,6 @@ apply plugin: 'java' apply plugin: 'application' -sourceCompatibility = '1.8' -targetCompatibility = '1.8' - repositories { mavenCentral() mavenLocal() diff --git a/tcbot-teamcity-ignited/build.gradle b/tcbot-teamcity-ignited/build.gradle index 41593b35e..8f6c5ebf3 100644 --- a/tcbot-teamcity-ignited/build.gradle +++ b/tcbot-teamcity-ignited/build.gradle @@ -16,12 +16,7 @@ */ apply plugin: 'java' - -repositories { - mavenCentral() - mavenLocal() -} - + dependencies { compile (project(":tcbot-teamcity")); compile (project(":tcbot-persistence")); diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java index eaeeb545f..c47d8e99b 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java @@ -554,6 +554,10 @@ public void addProblems(IStringCompactor compactor, .forEach(this.problems::add); } + public Long statisticValue(Integer propCode) { + return statistics == null ? null : statistics.statisticValue(propCode); + } + public Long buildDuration(IStringCompactor compactor) { return statistics == null ? null : statistics.buildDuration(compactor); } @@ -711,4 +715,8 @@ public int totalNotMutedTests() { return cnt; } + + public long getFinishDateTs() { + return finishDate; + } } diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java index 12a0f67bb..514542b58 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java @@ -29,6 +29,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; + /** * Statistics values to be saved in compacted form. */ @@ -85,10 +87,14 @@ public StatisticsCompacted(IStringCompactor compactor, Statistics statistics) { public Long buildDuration(IStringCompactor compactor) { Integer buildDurationId = compactor.getStringIdIfPresent(Statistics.BUILD_DURATION); - if (buildDurationId == null) + return statisticValue(buildDurationId); + } + + @Nullable public Long statisticValue(@Nullable Integer propCode) { + if (propCode == null) return null; - long val = findPropertyValue(buildDurationId); + long val = findPropertyValue(propCode); return val >= 0 ? val : null; } @@ -100,12 +106,7 @@ public Long buildDuration(IStringCompactor compactor) { public Long buildDurationNetTime(IStringCompactor compactor) { Integer buildDurationNetId = compactor.getStringIdIfPresent(Statistics.BUILD_DURATION_NET_TIME); - if (buildDurationNetId == null) - return null; - - long val = findPropertyValue(buildDurationNetId); - - return val >= 0 ? val : null; + return statisticValue(buildDurationNetId); } /** @@ -115,12 +116,7 @@ public Long buildDurationNetTime(IStringCompactor compactor) { public Long artifcactPublishingDuration(IStringCompactor compactor) { Integer buildDurationNetId = compactor.getStringIdIfPresent(Statistics.ARTIFACTS_PUBLISHING_DURATION); - if (buildDurationNetId == null) - return null; - - long val = findPropertyValue(buildDurationNetId); - - return val >= 0 ? val : null; + return statisticValue(buildDurationNetId); } /** @@ -130,12 +126,7 @@ public Long artifcactPublishingDuration(IStringCompactor compactor) { public Long dependeciesResolvingDuration(IStringCompactor compactor) { Integer buildDurationNetId = compactor.getStringIdIfPresent(Statistics.DEPENDECIES_RESOLVING_DURATION); - if (buildDurationNetId == null) - return null; - - long val = findPropertyValue(buildDurationNetId); - - return val >= 0 ? val : null; + return statisticValue(buildDurationNetId); } /** @@ -145,15 +136,10 @@ public Long dependeciesResolvingDuration(IStringCompactor compactor) { public Long sourceUpdateDuration(IStringCompactor compactor) { Integer buildDurationNetId = compactor.getStringIdIfPresent(Statistics.SOURCES_UPDATE_DURATION); - if (buildDurationNetId == null) - return null; - - long val = findPropertyValue(buildDurationNetId); - - return val >= 0 ? val : null; + return statisticValue(buildDurationNetId); } - private long findPropertyValue(int propCode) { + public long findPropertyValue(int propCode) { if (keys == null) return -1L; diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/InvocationData.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/InvocationData.java index f02d28679..87f354211 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/InvocationData.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/InvocationData.java @@ -27,6 +27,7 @@ import java.util.stream.Stream; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.tcbot.common.TcBotConst; import org.apache.ignite.tcbot.persistence.Persisted; import org.apache.ignite.tcignited.history.RunStatus; @@ -38,7 +39,7 @@ @Persisted public class InvocationData { /** Max days to keep test invocatoin data in run statistics: affects Bot Visa. */ - public static final int MAX_DAYS = 21; + public static final int MAX_DAYS = TcBotConst.HISTORY_MAX_DAYS; /** Muted. */ public static final int MUTED = RunStatus.RES_MUTED_FAILURE.getCode(); /** Failure. */ diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java index 4a0cd2ec1..b4b5d7986 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java @@ -22,6 +22,7 @@ import java.util.Objects; import java.util.stream.Collectors; import javax.annotation.Nullable; +import org.apache.ignite.tcbot.common.TcBotConst; import org.apache.ignite.tcbot.persistence.IVersionedEntity; import org.apache.ignite.tcbot.persistence.Persisted; import org.apache.ignite.tcignited.history.ChangesState; @@ -36,7 +37,6 @@ public class RunHistCompacted implements IVersionedEntity, IRunHistory { /** Latest version. */ private static final int LATEST_VERSION = 1; - public static final int FLAKYNESS_STATUS_CHANGE_BORDER = 1; /** Entity fields version. */ @SuppressWarnings("FieldCanBeLocal") @@ -92,7 +92,7 @@ public RunHistCompacted(RunHistKey ignored) { @Override public String getFlakyComments() { int statusChange = getStatusChangesWithoutCodeModification(); - if (statusChange < FLAKYNESS_STATUS_CHANGE_BORDER) + if (statusChange < TcBotConst.FLAKYNESS_STATUS_CHANGE_BORDER) return null; return "Test seems to be flaky: " + @@ -120,7 +120,7 @@ public int getStatusChangesWithoutCodeModification() { /** {@inheritDoc} */ @Override public boolean isFlaky() { - return getStatusChangesWithoutCodeModification() >= FLAKYNESS_STATUS_CHANGE_BORDER; + return getStatusChangesWithoutCodeModification() >= TcBotConst.FLAKYNESS_STATUS_CHANGE_BORDER; } /** {@inheritDoc} */ @@ -132,6 +132,7 @@ public int getStatusChangesWithoutCodeModification() { * @param inv Invocation. * @return if test run is new and is not expired. */ + @Deprecated public boolean addInvocation(Invocation inv) { return data.addInvocation(inv); } diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java index 91c485000..5ffd3155a 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java @@ -33,6 +33,7 @@ import org.apache.ignite.tcbot.common.conf.ITcServerConfig; import org.apache.ignite.tcignited.history.IRunHistory; import org.apache.ignite.tcignited.history.IRunStat; +import org.apache.ignite.tcignited.history.ISuiteRunHistory; import org.apache.ignite.tcservice.model.agent.Agent; import org.apache.ignite.tcservice.model.mute.MuteInfo; import org.apache.ignite.tcservice.model.result.Build; @@ -200,19 +201,22 @@ public default FatBuildCompacted getFatBuild(int id) { */ public BuildTypeCompacted getBuildType(String buildTypeId); + @Deprecated @Nullable public IRunHistory getTestRunHist(String testName, @Nullable String branch); + @Deprecated @Nullable public IRunHistory getSuiteRunHist(String suiteId, @Nullable String branch); + @Nullable public ISuiteRunHistory getSuiteRunHist(@Nullable Integer buildTypeId, @Nullable Integer normalizedBaseBranch); /** * V.3.0 run history implementation based on scan of fat builds. * * @param testName Test name. - * @param suiteName Suite name. - * @param branchName Branch name. + * @param buildTypeId Suite (Build Type) ID, ID for compactor. Null suite name means suite not found. + * @param normalizedBaseBranch Branch name. This branch name does not support branches equivalence, only exact query will work. */ - @Nullable public IRunHistory getTestRunHist(int testName, @Nullable Integer suiteName, @Nullable Integer branchName); + @Nullable public IRunHistory getTestRunHist(int testName, @Nullable Integer buildTypeId, @Nullable Integer normalizedBaseBranch); /** * @param suiteBuildTypeId Suite id. diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java index a6e37e5df..ce1050202 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java @@ -16,10 +16,8 @@ */ package org.apache.ignite.tcignited; -import com.google.common.collect.Lists; import com.google.common.collect.Sets; import java.io.File; -import java.time.Duration; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -32,10 +30,8 @@ import java.util.Objects; import java.util.OptionalInt; import java.util.Set; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Supplier; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -52,9 +48,6 @@ import org.apache.ignite.ci.teamcity.ignited.change.ChangeDao; import org.apache.ignite.ci.teamcity.ignited.change.ChangeSync; import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; -import org.apache.ignite.tcignited.mute.MuteDao; -import org.apache.ignite.tcignited.mute.MuteSync; -import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData; import org.apache.ignite.tcbot.common.conf.ITcServerConfig; import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; import org.apache.ignite.tcbot.common.interceptor.GuavaCached; @@ -64,13 +57,18 @@ import org.apache.ignite.tcignited.build.FatBuildDao; import org.apache.ignite.tcignited.build.ProactiveFatBuildSync; import org.apache.ignite.tcignited.buildlog.BuildLogCheckResultDao; +import org.apache.ignite.tcignited.buildref.BranchEquivalence; import org.apache.ignite.tcignited.buildref.BuildRefDao; import org.apache.ignite.tcignited.buildref.BuildRefSync; +import org.apache.ignite.tcignited.history.HistoryCollector; import org.apache.ignite.tcignited.history.IRunHistory; import org.apache.ignite.tcignited.history.IRunStat; +import org.apache.ignite.tcignited.history.ISuiteRunHistory; import org.apache.ignite.tcignited.history.RunHistCompactedDao; import org.apache.ignite.tcignited.history.RunHistSync; -import org.apache.ignite.tcservice.ITeamcity; +import org.apache.ignite.tcignited.history.SuiteInvocationHistoryDao; +import org.apache.ignite.tcignited.mute.MuteDao; +import org.apache.ignite.tcignited.mute.MuteSync; import org.apache.ignite.tcservice.ITeamcityConn; import org.apache.ignite.tcservice.model.agent.Agent; import org.apache.ignite.tcservice.model.conf.Project; @@ -95,11 +93,6 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { /** Max build id diff to enforce reload during incremental refresh. */ public static final int MAX_ID_DIFF_TO_ENFORCE_CONTINUE_SCAN = 3000; - /** Default synonyms. */ - private static final List DEFAULT_SYNONYMS - = Collections.unmodifiableList( - Lists.newArrayList(ITeamcity.DEFAULT, ITeamcity.REFS_HEADS_MASTER, ITeamcity.MASTER)); - /** Server (service) code. */ private String srvCode; @@ -151,11 +144,20 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { /** Run history sync. */ @Inject private RunHistSync runHistSync; - @Inject private BuildLogCheckResultDao logCheckResultDao; + /** Logger check result DAO. */ + @Inject private BuildLogCheckResultDao logCheckResDao; + + /** History DAO. */ + @Inject private SuiteInvocationHistoryDao histDao; + + /** History collector. */ + @Inject private HistoryCollector histCollector; /** Strings compactor. */ @Inject private IStringCompactor compactor; + @Inject private BranchEquivalence branchEquivalence; + /** Server ID mask for cache Entries. */ private int srvIdMaskHigh; @@ -172,7 +174,8 @@ public void init(ITeamcityConn conn) { changesDao.init(); runHistCompactedDao.init(); muteDao.init(); - logCheckResultDao.init(); + logCheckResDao.init(); + histDao.init(); } /** @@ -208,7 +211,8 @@ private String taskName(String taskName) { List buildRefs = getAllBuildsCompacted(buildTypeId, branchName) .stream() .filter(b -> b.isFinished(compactor)) - .filter(b -> b.status() != unknownStatus) + .filter(b -> b.status() != unknownStatus) //check build is not cancelled + .sorted(Comparator.comparing(BuildRefCompacted::id)) .collect(Collectors.toList()); if (buildRefs.isEmpty()) @@ -352,7 +356,7 @@ else if (midValStartDate.before(key)) @Nullable String branchName) { ensureActualizeRequested(); - return buildRefDao.getAllBuildsCompacted(srvIdMaskHigh, buildTypeId, branchForQuery(branchName)); + return buildRefDao.getAllBuildsCompacted(srvIdMaskHigh, buildTypeId, branchEquivalence.branchForQuery(branchName)); } /** {@inheritDoc} */ @@ -365,7 +369,7 @@ else if (midValStartDate.before(key)) if (stateQueuedId == null) return Collections.emptyList(); - Set branchNameIds = branchForQuery(branchName).stream().map(str -> compactor.getStringIdIfPresent(str)) + Set branchNameIds = branchEquivalence.branchForQuery(branchName).stream().map(str -> compactor.getStringIdIfPresent(str)) .filter(Objects::nonNull).collect(Collectors.toSet()); List res = new ArrayList<>(); @@ -422,7 +426,6 @@ else if (midValStartDate.before(key)) /** {@inheritDoc} */ @Nullable - @AutoProfiling @Override public IRunHistory getTestRunHist(String testName, @Nullable String branch) { return runHistCompactedDao.getTestRunHist(srvIdMaskHigh, testName, branch); } @@ -434,32 +437,27 @@ else if (midValStartDate.before(key)) return runHistCompactedDao.getSuiteRunHist(srvIdMaskHigh, suiteId, branch); } - @Nullable @Override - public IRunHistory getTestRunHist(int testName, @Nullable Integer suiteName, @Nullable Integer branchName) { - if (suiteName == null || branchName == null) + /** {@inheritDoc} */ + @Nullable @Override public ISuiteRunHistory getSuiteRunHist(@Nullable Integer buildTypeId, @Nullable Integer normalizedBaseBranch) { + if (buildTypeId == null || normalizedBaseBranch == null) return null; - Supplier> supplier = () -> { - String btId = compactor.getStringFromId(suiteName); - String branchId = compactor.getStringFromId(branchName); - List compacted = getAllBuildsCompacted(btId, branchId); - long curTs = System.currentTimeMillis(); - Set buildIds = compacted.stream().filter( - bRef -> { - Long startTime = getBuildStartTime(bRef.id()); - if (startTime == null) - return false; + if (buildTypeId < 0 || normalizedBaseBranch < 0) + return null; - return Duration.ofMillis(curTs - startTime).toDays() < InvocationData.MAX_DAYS; - } - ).map(BuildRefCompacted::id).collect(Collectors.toSet()); + return histCollector.getSuiteRunHist(srvIdMaskHigh, buildTypeId, normalizedBaseBranch); + } - System.err.println("Build " + btId + " branch " + branchId + " builds in scope " + buildIds.size()); + /** {@inheritDoc} */ + @Nullable @Override public IRunHistory getTestRunHist(int testName, @Nullable Integer buildTypeId, + @Nullable Integer normalizedBaseBranch) { + if (buildTypeId == null || normalizedBaseBranch == null) + return null; - return buildIds; - }; + if (testName < 0 || buildTypeId < 0 || normalizedBaseBranch < 0) + return null; - return fatBuildDao.getTestRunHist(srvIdMaskHigh, supplier, testName, suiteName, branchName); + return histCollector.getTestRunHist(srvIdMaskHigh, testName, buildTypeId, normalizedBaseBranch); } /** {@inheritDoc} */ @@ -504,13 +502,6 @@ public File downloadAndCacheBuildLog(int buildId) { return buildTypeDao.getFatBuildType(srvIdMaskHigh, buildTypeId); } - public List branchForQuery(@Nullable String branchName) { - if (ITeamcity.DEFAULT.equals(branchName)) - return DEFAULT_SYNONYMS; - else - return Collections.singletonList(branchName); - } - /** * Enables scheduling for build refs/builds/history sync */ @@ -522,7 +513,8 @@ public void ensureActualizeRequested() { // schedule find missing later fatBuildSync.ensureActualizationRequested(srvCode, conn); - runHistSync.invokeLaterFindMissingHistory(srvCode); + //todo remove unused code + // runHistSync.invokeLaterFindMissingHistory(srvCode); } /** {@inheritDoc} */ @@ -577,12 +569,17 @@ public Long getBuildStartTs(int buildId) { long ts = highBuild.getStartDateTs(); - return ts > 0 ? ts : null; + if (ts > 0) { + runHistCompactedDao.setBuildStartTime(srvIdMaskHigh, buildId, ts); + + return ts; + } else + return null; } - @GuavaCached(maximumSize = 100000, expireAfterAccessSecs = 90, softValues = true) + //@GuavaCached(maximumSize = 100000, expireAfterAccessSecs = 90, softValues = true) public Long getBuildStartTime(int buildId) { - return runHistCompactedDao.getBuildStartTime(srvIdMaskHigh, buildId); + return histCollector.getBuildStartTime(srvIdMaskHigh, buildId); } /** {@inheritDoc} */ diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java index 707cab8d2..bc0b20356 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java @@ -29,6 +29,8 @@ import org.apache.ignite.ci.teamcity.ignited.change.ChangeSync; import org.apache.ignite.tcignited.build.FatBuildDao; import org.apache.ignite.tcignited.build.ProactiveFatBuildSync; +import org.apache.ignite.tcignited.history.HistoryCollector; +import org.apache.ignite.tcignited.history.SuiteInvocationHistoryDao; import org.apache.ignite.tcignited.mute.MuteDao; import org.apache.ignite.tcignited.mute.MuteSync; import org.apache.ignite.tcignited.buildlog.BuildLogProcessorModule; @@ -65,6 +67,8 @@ public class TeamcityIgnitedModule extends AbstractModule { bind(MuteDao.class).in(new SingletonScope()); bind(MuteSync.class).in(new SingletonScope()); bind(BuildLogCheckResultDao.class).in(new SingletonScope()); + bind(SuiteInvocationHistoryDao.class).in(new SingletonScope()); + bind(HistoryCollector.class).in(new SingletonScope()); TcRealConnectionModule module = new TcRealConnectionModule(); if (conn != null) diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java index ad08fe334..5b4740d62 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java @@ -18,50 +18,40 @@ package org.apache.ignite.tcignited.build; import com.google.common.base.Preconditions; -import com.google.common.cache.CacheBuilder; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import javax.cache.Cache; -import javax.cache.processor.EntryProcessorException; -import javax.cache.processor.EntryProcessorResult; -import javax.cache.processor.MutableEntry; -import javax.inject.Inject; -import javax.inject.Provider; +import com.google.common.collect.Iterables; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; +import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.cache.CacheEntryProcessor; -import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted; import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; -import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted; -import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation; -import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted; -import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey; -import org.apache.ignite.tcbot.common.exeption.ExceptionUtil; import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; import org.apache.ignite.tcbot.persistence.CacheConfigs; import org.apache.ignite.tcbot.persistence.IStringCompactor; -import org.apache.ignite.tcignited.history.IRunHistory; +import org.apache.ignite.tcignited.buildref.BuildRefDao; +import org.apache.ignite.tcignited.buildtime.BuildTimeResult; +import org.apache.ignite.tcignited.history.HistoryCollector; import org.apache.ignite.tcservice.model.changes.ChangesList; +import org.apache.ignite.tcservice.model.hist.BuildRef; import org.apache.ignite.tcservice.model.result.Build; import org.apache.ignite.tcservice.model.result.problems.ProblemOccurrence; import org.apache.ignite.tcservice.model.result.stat.Statistics; -import org.apache.ignite.tcservice.model.result.tests.TestOccurrence; import org.apache.ignite.tcservice.model.result.tests.TestOccurrencesFull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.cache.Cache; +import javax.cache.processor.EntryProcessorException; +import javax.cache.processor.EntryProcessorResult; +import javax.cache.processor.MutableEntry; +import javax.inject.Inject; +import javax.inject.Provider; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + /** * */ @@ -71,6 +61,7 @@ public class FatBuildDao { /** Cache name */ public static final String TEAMCITY_FAT_BUILD_CACHE_NAME = "teamcityFatBuild"; + public static final int MAX_FAT_BUILD_CHUNK = 32 * 10; /** Ignite provider. */ @Inject private Provider igniteProvider; @@ -78,24 +69,11 @@ public class FatBuildDao { /** Builds cache. */ private IgniteCache buildsCache; - - /** Suite history cache. */ - private IgniteCache suiteHistory; - /** Compactor. */ @Inject private IStringCompactor compactor; - /** - * Non persistence cache for all suite RunHistory for particular branch. - * RunHistKey(ServerId||BranchId||suiteId)-> Build reference - */ - private final com.google.common.cache.Cache runHistInMemCache - = CacheBuilder.newBuilder() - .maximumSize(8000) - .expireAfterAccess(16, TimeUnit.MINUTES) - .softValues() - .build(); - + /** History collector. */ + @Inject private HistoryCollector histCollector; /** * @@ -155,7 +133,7 @@ public FatBuildDao init() { public void putFatBuild(int srvIdMaskHigh, int buildId, FatBuildCompacted newBuild) { buildsCache.put(buildIdToCacheKey(srvIdMaskHigh, buildId), newBuild); - invalidateHistoryInMem(srvIdMaskHigh, Stream.of(newBuild)); + histCollector.invalidateHistoryInMem(srvIdMaskHigh, newBuild); } public static int[] extractChangeIds(@Nonnull ChangesList changesList) { @@ -172,7 +150,7 @@ public static int[] extractChangeIds(@Nonnull ChangesList changesList) { } /** - * @param srvIdMaskHigh Server id mask high. + * @param srvIdMaskHigh Server id mask to be placed at high bits of the key. * @param buildId Build id. */ public static long buildIdToCacheKey(int srvIdMaskHigh, int buildId) { @@ -197,10 +175,7 @@ public FatBuildCompacted getFatBuild(int srvIdMaskHigh, int buildId) { public Map getAllFatBuilds(int srvIdMaskHigh, Collection buildsIds) { Preconditions.checkNotNull(buildsCache, "init() was not called"); - Set ids = buildsIds.stream() - .filter(Objects::nonNull) - .map(buildId -> buildIdToCacheKey(srvIdMaskHigh, buildId)) - .collect(Collectors.toSet()); + Set ids = buildsIdsToCacheKeys(srvIdMaskHigh, buildsIds); return buildsCache.getAll(ids); } @@ -223,108 +198,170 @@ public Stream> outdatedVersionEntries(int s .filter(entry -> isKeyForServer(entry.getKey(), srvId)); } - public IRunHistory getTestRunHist(int srvIdMaskHigh, - Supplier> buildIdsSupplier, int testName, int suiteName, int branchName) { - - - RunHistKey runHistKey = new RunHistKey(srvIdMaskHigh, suiteName, branchName); + private static Set buildsIdsToCacheKeys(int srvId, Collection stream) { + return stream.stream() + .filter(Objects::nonNull).map(id -> buildIdToCacheKey(srvId, id)).collect(Collectors.toSet()); + } - SuiteHistory history; + /** + * @param srvId Server id. + * @param buildId Build Id. + */ + @Nullable public Long getBuildStartTime(int srvId, Integer buildId) { + IgniteCache cacheBin = buildsCache.withKeepBinary(); + long key = buildIdToCacheKey(srvId, buildId); - try { - history = runHistInMemCache.get(runHistKey, - () -> { - Set buildIds = determineLatestBuilds(buildIdsSupplier); + return cacheBin.invoke(key, new GetStartTimeProc()); + } - return calcSuiteHistory(srvIdMaskHigh, buildIds); + /** + * @param srvId Server id. + * @param ids Ids. + */ + public Map getBuildStartTime(int srvId, Set ids) { + IgniteCache cacheBin = buildsCache.withKeepBinary(); + Set keys = buildsIdsToCacheKeys(srvId, ids); + HashMap res = new HashMap<>(); + + Iterables.partition(keys, MAX_FAT_BUILD_CHUNK).forEach( + chunk -> { + Map> map = cacheBin.invokeAll(keys, new GetStartTimeProc()); + map.forEach((k, r) -> { + Long ts = r.get(); + if (ts != null) + res.put(BuildRefDao.cacheKeyToBuildId(k), ts); }); - } - catch (ExecutionException e) { - throw ExceptionUtil.propagateException(e); - } + } + ); - return history.testsHistory.get(testName); + return res; } - @AutoProfiling - protected SuiteHistory calcSuiteHistory(int srvIdMaskHigh, Set buildIds) { - Set cacheKeys = buildIds.stream().map(id -> buildIdToCacheKey(srvIdMaskHigh, id)).collect(Collectors.toSet()); + public BuildTimeResult loadBuildTimeResult(int ageDays, List idsToCheck) { + int stateRunning = compactor.getStringId(BuildRef.STATE_RUNNING); + Integer buildDurationId = compactor.getStringIdIfPresent(Statistics.BUILD_DURATION); + int timeoutProblemCode = compactor.getStringId(ProblemOccurrence.TC_EXECUTION_TIMEOUT); - int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS); + BuildTimeResult res = new BuildTimeResult(); - CacheEntryProcessor> processor = new HistoryCollectProcessor(successStatusStrId); + // also may take affinity into account + Iterables.partition(idsToCheck, MAX_FAT_BUILD_CHUNK).forEach( + chunk -> { + HashSet keys = new HashSet<>(chunk); + Map all = buildsCache.getAll(keys); + all.forEach((key, build) -> { + if (build.isComposite()) + return; - Map>> map = buildsCache.invokeAll(cacheKeys, processor); + long runningTime = getBuildRunningTime(stateRunning, buildDurationId, build); + if (runningTime > 0) { + int buildTypeId = build.buildTypeId(); + System.err.println("Running " + runningTime + " BT: " + buildTypeId); - SuiteHistory hist = new SuiteHistory(); + int srvId = BuildRefDao.cacheKeyToSrvId(key); + boolean hasTimeout = build.hasBuildProblemType(timeoutProblemCode); - map.values().forEach( - res-> { - if(res==null) - return; + res.add(srvId, buildTypeId, runningTime, hasTimeout); + } + }); + } + ); - Map invocationMap = res.get(); + return res; - if(invocationMap == null) - return; + } - invocationMap.forEach((k, v) -> { - RunHistCompacted compacted = hist.testsHistory.computeIfAbsent(k, - k_ -> new RunHistCompacted()); + public static long getBuildRunningTime(int stateRunning, Integer buildDurationId, + FatBuildCompacted buildBinary) { + long startTs = buildBinary.getStartDateTs(); - compacted.innerAddInvocation(v); - }); + if (startTs <= 0) + return -1; + int state = buildBinary.state(); + + long runningTime = -1; + if (stateRunning == state) + runningTime = System.currentTimeMillis() - startTs; + + if (runningTime < 0) { + if (buildDurationId != null) { + Long val = buildBinary.statisticValue(buildDurationId); + + runningTime = (val != null && val >= 0) ? val : -1; } - ); - System.err.println("Suite history: tests in scope " - + hist.testsHistory.size() - + " for " +buildIds.size() + " builds checked" - + " size " + hist.size(igniteProvider.get())); - return hist; - } + } - @AutoProfiling - protected Set determineLatestBuilds(Supplier> buildIdsSupplier) { - return buildIdsSupplier.get(); - } + if (runningTime < 0) { + long finishTs = buildBinary.getFinishDateTs(); - public void invalidateHistoryInMem(int srvId, Stream stream) { - Iterable objects = - stream - .map(b -> new RunHistKey(srvId, b.buildTypeId(), b.branchName())) - .collect(Collectors.toSet()); + if (finishTs > 0) + runningTime = finishTs - startTs; + } - runHistInMemCache.invalidateAll(objects); + return runningTime; } + public static long getBuildRunningTime(int stateRunning, Integer buildDurationId, + BinaryObject buildBinary) { + Long startTs = buildBinary.field("startDate"); + + if (startTs == null || startTs <= 0) + return -1; + + + int status = buildBinary.field("status"); + int state = buildBinary.field("state"); + + long runningTime = -1; + if(stateRunning == state) + runningTime = System.currentTimeMillis() - startTs; + + if(runningTime<0){ + + if (buildDurationId != null) { + BinaryObject statistics = buildBinary.field("statistics"); + if(statistics!=null) { + // statistics.field() + } + long val = -1; //statistics.findPropertyValue(buildDurationId); + + runningTime = val >= 0 ? val : -1; + } - private static class HistoryCollectProcessor implements CacheEntryProcessor> { - private final int successStatusStrId; - public HistoryCollectProcessor(int successStatusStrId) { - this.successStatusStrId = successStatusStrId; } - @Override public Map process(MutableEntry entry, + if(runningTime<0) { + Long finishTs= buildBinary.field("finishDate"); + + if(finishTs!=null) + runningTime = finishTs - startTs; + } + + return runningTime; + } + + private static class GetStartTimeProc implements CacheEntryProcessor { + public GetStartTimeProc() { + } + + /** {@inheritDoc} */ + @Override public Long process(MutableEntry entry, Object... arguments) throws EntryProcessorException { if (entry.getValue() == null) return null; - Map hist = new HashMap<>(); - FatBuildCompacted fatBuildCompacted = entry.getValue(); - Stream tests = fatBuildCompacted.getAllTests(); - tests.forEach( - testCompacted -> { - Invocation invocation = testCompacted.toInvocation(fatBuildCompacted, (k, v) -> false, successStatusStrId); + BinaryObject buildBinary = entry.getValue(); - hist.put(testCompacted.testName(), invocation); - } - ); + Long startDate = buildBinary.field("startDate"); + + if (startDate == null || startDate <= 0) + return null; - return hist; + return startDate; } } } diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/ProactiveFatBuildSync.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/ProactiveFatBuildSync.java index 3e01c05c7..8ad06a7d3 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/ProactiveFatBuildSync.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/ProactiveFatBuildSync.java @@ -315,7 +315,8 @@ public FatBuildCompacted loadBuild(ITeamcityConn conn, int buildId, buildRefDao.save(srvIdMask, refCompacted); - runHistSync.saveToHistoryLater(srvCode, savedVer); + //todo remove unused code + // runHistSync.saveToHistoryLater(srvCode, savedVer); return savedVer; } diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java index 896973657..0e2d010e2 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java @@ -21,15 +21,50 @@ import java.util.Map; import org.apache.ignite.Ignite; +import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation; import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted; import org.apache.ignite.internal.binary.BinaryObjectExImpl; +import org.apache.ignite.tcignited.history.IRunHistory; +import org.apache.ignite.tcignited.history.ISuiteRunHistory; +import org.apache.ignite.tcignited.history.SuiteInvocation; -public class SuiteHistory { +/** + * Suite run history summary. + */ +public class SuiteHistory implements ISuiteRunHistory { /** Tests history: Test name ID->RunHistory */ - Map testsHistory = new HashMap<>(); + private Map testsHistory = new HashMap<>(); + + private RunHistCompacted suiteHist = new RunHistCompacted(); public int size(Ignite ignite) { BinaryObjectExImpl binary = ignite.binary().toBinary(this); return binary.length(); } + + public IRunHistory getTestRunHist(int testName) { + return testsHistory.get(testName); + } + + public RunHistCompacted getOrAddTestsHistory(Integer tName) { + return testsHistory.computeIfAbsent(tName, k_ -> new RunHistCompacted()); + } + + public void addTestInvocation(Integer tName, Invocation invocation) { + getOrAddTestsHistory(tName).innerAddInvocation(invocation); + } + + public void addSuiteInvocation(SuiteInvocation suiteInv) { + suiteInv.tests().forEach(this::addTestInvocation); + + suiteHist.innerAddInvocation(suiteInv.suiteInvocation()); + } + + public RunHistCompacted getSuiteHist() { + return suiteHist; + } + + @Override public IRunHistory self() { + return suiteHist; + } } diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BranchEquivalence.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BranchEquivalence.java new file mode 100644 index 000000000..ab5f0b9ed --- /dev/null +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BranchEquivalence.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.tcignited.buildref; + +import com.google.common.collect.Lists; +import java.util.Collections; +import java.util.List; +import javax.annotation.Nullable; +import org.apache.ignite.tcservice.ITeamcity; + +public class BranchEquivalence { + /** Default synonyms. */ + private static final List DEFAULT_SYNONYMS + = Collections.unmodifiableList( + Lists.newArrayList(ITeamcity.DEFAULT, ITeamcity.REFS_HEADS_MASTER, ITeamcity.MASTER)); + + + public List branchForQuery(@Nullable String branchName) { + if (ITeamcity.DEFAULT.equals(branchName)) + return DEFAULT_SYNONYMS; + else + return Collections.singletonList(branchName); + } + +} diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java index f3a3e89c0..ffad6004d 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java @@ -36,13 +36,17 @@ import javax.inject.Provider; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; +import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.cache.QueryEntity; import org.apache.ignite.cache.query.QueryCursor; +import org.apache.ignite.cache.query.ScanQuery; import org.apache.ignite.cache.query.SqlQuery; import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted; import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.util.GridIntList; +import org.apache.ignite.lang.IgniteCallable; +import org.apache.ignite.resources.IgniteInstanceResource; import org.apache.ignite.tcbot.common.exeption.ExceptionUtil; import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; import org.apache.ignite.tcbot.common.interceptor.GuavaCached; @@ -104,7 +108,7 @@ public Stream compactedBuildsForServer(int srvId) { * @param srvId Server id. */ public static boolean isKeyForServer(Long key, int srvId) { - return key!=null && key >> 32 == srvId; + return key!=null && cacheKeyToSrvId(key) == srvId; } /** @@ -168,6 +172,13 @@ public static int cacheKeyToBuildId(Long cacheKey) { return (int)(l >> 32); } + /** + * @param cacheKey Cache key. + */ + public static int cacheKeyToSrvId(long cacheKey) { + return (int)(cacheKey >> 32); + } + /** * @param srvId Server id mask high. * @param buildTypeId Build type (suite) id. @@ -175,7 +186,7 @@ public static int cacheKeyToBuildId(Long cacheKey) { */ @AutoProfiling @Nonnull public List getAllBuildsCompacted(int srvId, - String buildTypeId, + String buildTypeId, List bracnhNameQry) { Integer buildTypeIdId = compactor.getStringIdIfPresent(buildTypeId); if (buildTypeIdId == null) @@ -307,4 +318,8 @@ public int[] getAllIds(int srvId) { return StreamSupport.stream(buildRefsCache.spliterator(), false) .filter(entry -> isKeyForServer(entry.getKey(), srvId)); } + + public IgniteCache buildRefsCache() { + return buildRefsCache; + } } diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeRecord.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeRecord.java new file mode 100644 index 000000000..63e80266a --- /dev/null +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeRecord.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.tcignited.buildtime; + +public class BuildTimeRecord { + private long totaltime; + private int cnt; + + public void addInvocation(long runningTimeMs) { + totaltime += runningTimeMs; + cnt++; + } + + public long avgDuration() { + if (cnt == 0) + return 0; + + return totaltime / cnt; + } + + public long totalDuration() { + return totaltime; + } +} diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeResult.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeResult.java new file mode 100644 index 000000000..c5a68df4d --- /dev/null +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeResult.java @@ -0,0 +1,65 @@ +package org.apache.ignite.tcignited.buildtime; + +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class BuildTimeResult { + private Map btByBuildType = new HashMap<>(); + private Map timedOutByBuildType = new HashMap<>(); + + public void add(int srvId, int buildTypeId, long runningTime, boolean hasTimeout) { + long cacheKey = buildTypeToCacheKey(srvId, buildTypeId); + btByBuildType.computeIfAbsent(cacheKey, k -> new BuildTimeRecord()).addInvocation(runningTime); + + if (hasTimeout) + timedOutByBuildType.computeIfAbsent(cacheKey, k -> new BuildTimeRecord()).addInvocation(runningTime); + } + + public static long buildTypeToCacheKey(long srvId, int btId) { + return (long)btId | srvId << 32; + } + + public static int cacheKeyToSrvId(long cacheKey) { + return (int)(cacheKey >> 32); + } + + public static int cacheKeyToBuildType(Long cacheKey) { + long l = cacheKey << 32; + return (int)(l >> 32); + } + + + public List> topByBuildTypes(Set availableServers, + long minAvgDurationMs, + int maxCnt) { + return filtered(btByBuildType, availableServers, minAvgDurationMs) + .sorted(Comparator.comparing( + (Function, Long>) entry -> entry.getValue().avgDuration()) + .reversed()) + .limit(maxCnt) + .collect(Collectors.toList()); + } + + public List> topTimeoutsByBuildTypes(Set availableServers, + long minAvgDurationMs, + int maxCnt) { + return filtered(timedOutByBuildType, availableServers, minAvgDurationMs) + .sorted(Comparator.comparing( + (Function, Long>) entry -> entry.getValue().avgDuration()) + .reversed()) + .limit(maxCnt) + .collect(Collectors.toList()); + } + + private Stream> filtered(Map map, Set availableServers, long minAvgDurationMs) { + return map.entrySet().stream() + .filter(e -> { + Long key = e.getKey(); + int srvId = cacheKeyToSrvId(key); + return availableServers.contains(srvId); + }) + .filter(e -> e.getValue().avgDuration() > minAvgDurationMs); + } +} diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java new file mode 100644 index 000000000..1246a3eb8 --- /dev/null +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java @@ -0,0 +1,428 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.tcignited.history; + +import com.google.common.cache.CacheBuilder; +import com.google.common.collect.Iterables; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.binary.BinaryObject; +import org.apache.ignite.cache.query.QueryCursor; +import org.apache.ignite.cache.query.ScanQuery; +import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted; +import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted; +import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation; +import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey; +import org.apache.ignite.tcbot.common.TcBotConst; +import org.apache.ignite.tcbot.common.exeption.ExceptionUtil; +import org.apache.ignite.tcbot.common.exeption.ServicesStartingException; +import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; +import org.apache.ignite.tcbot.persistence.IStringCompactor; +import org.apache.ignite.tcignited.ITeamcityIgnited; +import org.apache.ignite.tcignited.build.FatBuildDao; +import org.apache.ignite.tcignited.build.SuiteHistory; +import org.apache.ignite.tcignited.buildref.BranchEquivalence; +import org.apache.ignite.tcignited.buildref.BuildRefDao; +import org.apache.ignite.tcservice.model.hist.BuildRef; +import org.apache.ignite.tcservice.model.result.tests.TestOccurrence; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.cache.Cache; +import javax.inject.Inject; +import javax.inject.Provider; +import java.time.Duration; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiPredicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * + */ +public class HistoryCollector { + /** Logger. */ + private static final Logger logger = LoggerFactory.getLogger(HistoryCollector.class); + + /** History DAO. */ + @Inject private SuiteInvocationHistoryDao histDao; + + /** Fat build DAO. */ + @Inject private FatBuildDao fatBuildDao; + + /** Build reference DAO. */ + @Inject private BuildRefDao buildRefDao; + + /** Compactor. */ + @Inject private IStringCompactor compactor; + + /** Ignite provider. */ + @Inject private Provider igniteProvider; + + /** Branch equivalence. */ + @Inject private BranchEquivalence branchEquivalence; + + /** Run history DAO. */ + @Inject private RunHistCompactedDao runHistCompactedDao; + + /** + * Non persistence cache for all suite RunHistory for particular branch. RunHistKey(ServerId||BranchId||suiteId)-> + * Build reference + */ + private final com.google.common.cache.Cache runHistInMemCache + = CacheBuilder.newBuilder() + .maximumSize(8000) + .expireAfterAccess(16, TimeUnit.MINUTES) + .softValues() + .build(); + + /** Biggest build ID, which out of history scope (MAX days + 2). */ + private final ConcurrentMap biggestBuildIdOutOfHistoryScope = new ConcurrentHashMap<>(); + + /** + * @param srvIdMaskHigh Server id mask to be placed at high bits in the key. + * @param testName Test name. + * @param buildTypeId Suite (Build type) id. + * @param normalizedBaseBranch Branch name. + */ + public IRunHistory getTestRunHist(int srvIdMaskHigh, int testName, int buildTypeId, + int normalizedBaseBranch) { + + SuiteHistory hist = getSuiteHist(srvIdMaskHigh, buildTypeId, normalizedBaseBranch); + + return hist.getTestRunHist(testName); + } + + @AutoProfiling + protected SuiteHistory getSuiteHist(int srvIdMaskHigh, int buildTypeId, int normalizedBaseBranch) { + RunHistKey runHistKey = new RunHistKey(srvIdMaskHigh, buildTypeId, normalizedBaseBranch); + + SuiteHistory hist; + try { + hist = runHistInMemCache.get(runHistKey, + () -> loadSuiteHistory(srvIdMaskHigh, buildTypeId, normalizedBaseBranch)); + } + catch (ExecutionException e) { + throw ExceptionUtil.propagateException(e); + } + + return hist; + } + + /** + * Latest actual Build ids supplier. This supplier should handle all equivalent branches in + * it. + * @param srvId + * @param buildTypeId + * @param normalizedBaseBranch + * @param knownBuilds Known builds, which already present in run history. + */ + @AutoProfiling + protected Set determineLatestBuilds( + int srvId, int buildTypeId, int normalizedBaseBranch, Set knownBuilds) { + String btId = compactor.getStringFromId(buildTypeId); + String branchId = compactor.getStringFromId(normalizedBaseBranch); + List bRefsList = buildRefDao.getAllBuildsCompacted(srvId, btId, + branchEquivalence.branchForQuery(branchId)); + + long curTs = System.currentTimeMillis(); + Set buildIds = bRefsList.stream() + .filter(b -> { + /* AtomicInteger biggestIdOutOfScope = biggestBuildIdOutOfHistoryScope.get(srvId); + int outOfScopeBuildId = biggestIdOutOfScope == null ? -1 : biggestIdOutOfScope.get(); + return b.id() > outOfScopeBuildId; + */ + + Integer maxBuildIdForDay = runHistCompactedDao.getBorderForAgeForBuildId(srvId, TcBotConst.HISTORY_BUILD_ID_BORDER_DAYS); + + if (maxBuildIdForDay == null) + return true; + + return b.id()>maxBuildIdForDay; + + }) + .filter(this::applicableForHistory) + .map(BuildRefCompacted::id) + .filter(bId -> !knownBuilds.contains(bId)).collect(Collectors.toSet()); + + System.out.println("***** Loading build start time history for suite " + + compactor.getStringFromId(buildTypeId) + + " branch " + compactor.getStringFromId(normalizedBaseBranch) + ": " + buildIds.size() + " builds" ); + + Map buildStartTimes = getStartTimeFromSpecialCache(srvId, buildIds); + + Set notFoundKeys = new HashSet<>(buildIds); + notFoundKeys.removeAll(buildStartTimes.keySet()); + + if (!notFoundKeys.isEmpty()) { + Map buildStartTimeFromFatBuild = getStartTimeFromFatBuild(srvId, notFoundKeys); + + buildStartTimes.putAll(buildStartTimeFromFatBuild); + + runHistCompactedDao.setBuildsStartTime(srvId, buildStartTimeFromFatBuild); + } + + Set buildInScope = buildIds.stream().filter( + bId -> { + Long startTime = buildStartTimes.get(bId); + if (startTime == null) + return false; + + long ageInDays = Duration.ofMillis(curTs - startTime).toDays(); + + if (ageInDays > TcBotConst.HISTORY_BUILD_ID_BORDER_DAYS) { + AtomicInteger integer = biggestBuildIdOutOfHistoryScope.computeIfAbsent(srvId, + s -> { + AtomicInteger atomicInteger = new AtomicInteger(); + atomicInteger.set(-1); + return atomicInteger; + }); + + int newBorder = integer.accumulateAndGet(bId, Math::max); + + if (newBorder == bId) + logger.info("History Collector: New border for server was set " + bId); + } + + return ageInDays < TcBotConst.HISTORY_MAX_DAYS; + } + ).collect(Collectors.toSet()); + + System.err.println("*** Build " + btId + " branch " + branchId + " builds in scope " + + buildInScope.size() + " from " + bRefsList.size()); + + return buildInScope; + } + + @SuppressWarnings("WeakerAccess") + @AutoProfiling + protected Map getStartTimeFromSpecialCache(int srvId, Set buildIds) { + return runHistCompactedDao.getBuildsStartTime(srvId, buildIds); + } + + @SuppressWarnings("WeakerAccess") + @AutoProfiling + protected Map getStartTimeFromFatBuild(int srvId, Set buildIds) { + return fatBuildDao.getBuildStartTime(srvId, buildIds); + } + + /** + * @param ref Build Reference or fat build. + */ + private boolean applicableForHistory(BuildRefCompacted ref) { + return !ref.isFakeStub() && !ref.isCancelled(compactor) && ref.isFinished(compactor); + } + + @AutoProfiling + protected SuiteHistory loadSuiteHistory(int srvId, + int buildTypeId, + int normalizedBaseBranch) { + Map suiteRunHist = histDao.getSuiteRunHist(srvId, buildTypeId, normalizedBaseBranch); + + logger.info("***** Found history for suite " + + compactor.getStringFromId(buildTypeId) + + " branch " + compactor.getStringFromId(normalizedBaseBranch) + ": " + suiteRunHist.size() ); + + Set buildIds = determineLatestBuilds(srvId, buildTypeId, normalizedBaseBranch, suiteRunHist.keySet()); + + HashSet missedBuildsIds = new HashSet<>(buildIds); + + missedBuildsIds.removeAll(suiteRunHist.keySet()); + + if (!missedBuildsIds.isEmpty()) { + Map addl = addSuiteInvocationsToHistory(srvId, missedBuildsIds, normalizedBaseBranch); + + suiteRunHist.putAll(addl); + + /* + Map reloaded = histDao.getSuiteRunHist(srvId, buildTypeId, normalizedBaseBranch); + + addl.keySet().forEach((k) -> { + Preconditions.checkState( reloaded.containsKey(k)); + }); + */ + } + + SuiteHistory sumary = new SuiteHistory(); + + suiteRunHist.forEach((buildId, suiteInv) -> sumary.addSuiteInvocation(suiteInv)); + + if (logger.isDebugEnabled()) { + logger.debug("***** History for suite " + + compactor.getStringFromId(buildTypeId) + + " branch" + compactor.getStringFromId(normalizedBaseBranch) + " requires " + + sumary.size(igniteProvider.get()) + " bytes"); + } + + return sumary; + } + + /** + * @param srvId Server id. + * @param b Build ref to invalidate. + */ + public void invalidateHistoryInMem(int srvId, BuildRefCompacted b) { + RunHistKey inv = new RunHistKey(srvId, b.buildTypeId(), b.branchName()); + + runHistInMemCache.invalidate(inv); + } + + + @AutoProfiling + protected Map addSuiteInvocationsToHistory(int srvId, + HashSet missedBuildsIds, int normalizedBaseBranch) { + Map suiteRunHist = new HashMap<>(); + int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS); + + System.err.println(Thread.currentThread().getName() + ": GET ALL: " + missedBuildsIds.size()); + + Iterables.partition(missedBuildsIds, 32 * 10).forEach( + chunk -> { + fatBuildDao.getAllFatBuilds(srvId, chunk).forEach((buildCacheKey, fatBuildCompacted) -> { + if (!applicableForHistory(fatBuildCompacted)) + return; + + BiPredicate paramsFilter = (k, v) -> false; + + SuiteInvocation sinv = new SuiteInvocation(srvId, normalizedBaseBranch, fatBuildCompacted, compactor, paramsFilter); + + Stream tests = fatBuildCompacted.getAllTests(); + tests.forEach( + testCompacted -> { + Invocation invocation = testCompacted.toInvocation(fatBuildCompacted, paramsFilter, successStatusStrId); + + sinv.addTest(testCompacted.testName(), invocation); + } + ); + + suiteRunHist.put(fatBuildCompacted.id(), sinv); + }); + } + ); + + + System.err.println("***** + Adding to persisted history " + + " branch " + compactor.getStringFromId(normalizedBaseBranch) + ": added " + + suiteRunHist.size() + " invocations from " + missedBuildsIds.size() + " builds checked"); + + histDao.putAll(srvId, suiteRunHist); + + return suiteRunHist; + } + + /** + * @param srvId Server id. + * @param buildTypeId Build type id. + * @param normalizedBaseBranch Normalized base branch. + */ + public ISuiteRunHistory getSuiteRunHist(int srvId, int buildTypeId, int normalizedBaseBranch) { + return getSuiteHist(srvId, buildTypeId, normalizedBaseBranch); + } + + + /** + * @param srvId Server id. + * @param buildId Build id. + */ + public Long getBuildStartTime(int srvId, int buildId) { + Long time = runHistCompactedDao.getBuildStartTime(srvId, buildId); + + if (time != null) + return time; + + time = fatBuildDao.getBuildStartTime(srvId, buildId); + + if (time != null) + runHistCompactedDao.setBuildStartTime(srvId, buildId, time); + + return time; + } + + public List findAllRecentBuilds(int days, Collection allServers) { + IgniteCache cache = buildRefDao.buildRefsCache(); + if (cache == null) + throw new ServicesStartingException(new RuntimeException("Ignite is not yet available")); + + IgniteCache cacheBin = cache.withKeepBinary(); + + final Map preBorder = new HashMap<>(); + + allServers.stream() + .map(ITeamcityIgnited::serverIdToInt) + .forEach(srvId -> { + Integer borderForAgeForBuildId = runHistCompactedDao.getBorderForAgeForBuildId(srvId, days); + if (borderForAgeForBuildId != null) + preBorder.put(srvId, borderForAgeForBuildId); + }); + + final int stateQueued = compactor.getStringId(BuildRef.STATE_QUEUED); + + long minTs = System.currentTimeMillis() - Duration.ofDays(days).toMillis(); + QueryCursor> query = cacheBin.query( + new ScanQuery() + .setFilter((key, v) -> { + int srvId = BuildRefDao.cacheKeyToSrvId(key); + Integer buildIdBorder = preBorder.get(srvId); + if (buildIdBorder != null) { + int id = v.field("id"); + if (id < buildIdBorder) + return false;// pre-filtered build out of scope + } + int state = v.field("state"); + + return stateQueued != state; + })); + + int cnt = 0; + List idsToCheck = new ArrayList<>(); + + try (QueryCursor> cursor = query) { + for (Cache.Entry next : cursor) { + Long key = next.getKey(); + int srvId = BuildRefDao.cacheKeyToSrvId(key); + + int buildId = BuildRefDao.cacheKeyToBuildId(key); + + Integer borderBuildId = runHistCompactedDao.getBorderForAgeForBuildId(srvId, days); + + boolean passesDate = borderBuildId == null || buildId >= borderBuildId; + + if (!passesDate) + continue; + + Long startTs = getBuildStartTime(srvId, buildId); + if (startTs == null || startTs < minTs) + continue; //time not saved in the DB, skip + + System.err.println("Found build at srv [" + srvId + "]: [" + buildId + "] to analyze, ts=" + startTs); + + cnt++; + + idsToCheck.add(key); + } + } + + System.err.println("Total builds to load " + cnt); + + return idsToCheck; + } +} diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/ISuiteRunHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/ISuiteRunHistory.java new file mode 100644 index 000000000..38dff1c0d --- /dev/null +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/ISuiteRunHistory.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.tcignited.history; + +public interface ISuiteRunHistory { + IRunHistory self(); + IRunHistory getTestRunHist(int testName); +} diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java index 8fed725b6..aa49966c0 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java @@ -17,9 +17,19 @@ package org.apache.ignite.tcignited.history; +import java.time.Duration; +import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicIntegerArray; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.cache.Cache; @@ -36,15 +46,17 @@ import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted; import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey; import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.tcbot.common.TcBotConst; import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; import org.apache.ignite.tcbot.common.interceptor.GuavaCached; import org.apache.ignite.tcbot.persistence.CacheConfigs; import org.apache.ignite.tcbot.persistence.IStringCompactor; +import org.apache.ignite.tcignited.buildref.BuildRefDao; import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch; /** - * + * TODO: rename to build start time storage */ public class RunHistCompactedDao { /** Cache name. */ @@ -61,14 +73,27 @@ public class RunHistCompactedDao { private Provider igniteProvider; /** Test history cache. */ + @Deprecated private IgniteCache testHistCache; /** Suite history cache. */ + @Deprecated private IgniteCache suiteHistCache; /** Build start time. */ private IgniteCache buildStartTime; + /** + * Biggest build ID, which is older than particular days count. + * Map: server ID-> Array of build Ids + * Array[0] = max build ID + * Array[1] = max build ID older than 1 day + */ + private final ConcurrentMap maxBuildIdOlderThanDays = new ConcurrentHashMap<>(); + + /** Millis in day. */ + private static final long MILLIS_IN_DAY = Duration.ofDays(1).toMillis(); + /** Compactor. */ @Inject private IStringCompactor compactor; @@ -133,15 +158,36 @@ public boolean buildWasProcessed(int srvId, int buildId) { */ @AutoProfiling @Nullable public Long getBuildStartTime(int srvId, int buildId) { - return buildStartTime.get(buildIdToCacheKey(srvId, buildId)); + Long ts = buildStartTime.get(buildIdToCacheKey(srvId, buildId)); + if (ts == null || ts <= 0) + return null; + + processBuildForBorder(srvId, buildId, ts); + + return ts; + } + + public boolean setBuildStartTime(int srvId, int buildId, long ts) { + if (ts <= 0) + return false; + + processBuildForBorder(srvId, buildId, ts); + + return buildStartTime.putIfAbsent(buildIdToCacheKey(srvId, buildId), ts); } @AutoProfiling public boolean setBuildProcessed(int srvId, int buildId, long ts) { + if (ts <= 0) + return false; + + processBuildForBorder(srvId, buildId, ts); + return buildStartTime.putIfAbsent(buildIdToCacheKey(srvId, buildId), ts); } @AutoProfiling + @Deprecated public Integer addTestInvocations(RunHistKey histKey, List list) { if (list.isEmpty()) return 0; @@ -234,4 +280,74 @@ public void disableWal() { cluster.disableWal(testHistCache.getName()); cluster.disableWal(suiteHistCache.getName()); } + + private static Set buildsIdsToCacheKeys(int srvId, Collection ids) { + return ids.stream() + .filter(Objects::nonNull).map(id -> buildIdToCacheKey(srvId, id)).collect(Collectors.toSet()); + } + + public Map getBuildsStartTime(int srvId, Set ids) { + Set cacheKeys = buildsIdsToCacheKeys(srvId, ids); + + Map res = new HashMap<>(); + + buildStartTime.getAll(cacheKeys).forEach((k, ts) -> { + if (ts != null && ts > 0) { + int buildId = BuildRefDao.cacheKeyToBuildId(k); + + res.put(buildId, ts); + + processBuildForBorder(srvId, buildId, ts); + } + }); + + return res; + } + + public void setBuildsStartTime(int srvId, Map builds) { + Map res = new HashMap<>(); + + builds.forEach((buildId, ts) -> { + if (ts != null && ts > 0) { + res.put(buildIdToCacheKey(srvId, buildId), ts); + + processBuildForBorder(srvId, buildId, ts); + } + }); + + buildStartTime.putAll(res); + } + + private void processBuildForBorder(int srvId, Integer buildId, Long ts) { + if (ts == null || ts <= 0) + return; + + AtomicIntegerArray arr = maxBuildIdOlderThanDays.computeIfAbsent(srvId, + k -> new AtomicIntegerArray(TcBotConst.BUILD_MAX_DAYS + 1)); + + long ageMs = System.currentTimeMillis() - ts; + if (ageMs < 0) + return; + + long days = ageMs / MILLIS_IN_DAY; + if (days > TcBotConst.BUILD_MAX_DAYS) + days = TcBotConst.BUILD_MAX_DAYS; + + arr.accumulateAndGet((int)days, buildId, Math::max); + } + + @Nullable public Integer getBorderForAgeForBuildId(int srvId, int ageDays) { + AtomicIntegerArray arr = maxBuildIdOlderThanDays.get(srvId); + if (arr == null) + return null; + + for (int i = ageDays; i < TcBotConst.BUILD_MAX_DAYS; i++) { + int buildId = arr.get(i); + if (buildId != 0) + return buildId; + } + + return null; + } + } diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java index a0a90f44b..987473f5b 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistSync.java @@ -104,6 +104,7 @@ public static String normalizeBranch(@Nullable String branchName) { * @param srvCode Server code (internal identification). * @param build Build. */ + @Deprecated public void saveToHistoryLater(String srvCode, FatBuildCompacted build) { if (!validForStatistics(build)) return; @@ -201,10 +202,13 @@ protected String saveBuildToHistory(String srvName, int ldrToActivate) { return saveInvocationsMap(buildsSaveThisRun, testsSaveThisRun); } + @Deprecated @AutoProfiling @Nonnull protected String saveInvocationsMap( Map> buildsSaveThisRun, Map> testsSaveThisRun) { + if (Boolean.valueOf(System.getProperty(TcBotSystemProperties.DEV_MODE))) + return "Skipped"; if (Boolean.valueOf(System.getProperty(TcBotSystemProperties.DEV_MODE))) if (testsSaveThisRun.size() > 100) @@ -257,6 +261,7 @@ protected String saveBuildToHistory(String srvName, int ldrToActivate) { return res; } + @Deprecated private void saveInvocationList(Set confirmedNewBuild, Set confirmedDuplicate, AtomicInteger invocations, @@ -295,6 +300,7 @@ private void saveInvocationList(Set confirmedNewBuild, duplicateOrExpired.addAndGet(invocationList.size() - cntAdded); } + @Deprecated public void invokeLaterFindMissingHistory(String srvName) { scheduler.sheduleNamed(taskName("findMissingHistFromBuildRef", srvName), () -> findMissingHistFromBuildRef(srvName), 12, TimeUnit.HOURS); @@ -308,6 +314,7 @@ private String taskName(String taskName, String srvName) { @SuppressWarnings({"WeakerAccess", "UnusedReturnValue"}) @MonitoredTask(name = "Find Missing Build History", nameExtArgsIndexes = {0}) @AutoProfiling + @Deprecated protected String findMissingHistFromBuildRef(String srvId) { int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId); @@ -340,6 +347,7 @@ protected String findMissingHistFromBuildRef(String srvId) { * @param srvNme Server name; * @param load Build IDs to be loaded into history cache later. */ + @Deprecated private void scheduleHistLoad(String srvNme, List load) { load.forEach(id -> { FatBuildCompacted fatBuild = fatBuildDao.getFatBuild(ITeamcityIgnited.serverIdToInt(srvNme), id); diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java new file mode 100644 index 000000000..7342af769 --- /dev/null +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.tcignited.history; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.function.BiPredicate; +import org.apache.ignite.cache.affinity.AffinityKeyMapped; +import org.apache.ignite.cache.query.annotations.QuerySqlField; +import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; +import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation; +import org.apache.ignite.tcbot.persistence.IStringCompactor; +import org.apache.ignite.tcbot.persistence.Persisted; + +/** + * Shorter verison of FatBuild with less data: created only if run history was required, + * has time limitation of MAX_DAYS, may have TTL. + */ +@Persisted +public class SuiteInvocation { + /** Server ID for queries */ + @QuerySqlField(orderedGroups = {@QuerySqlField.Group(name = "serverSuiteBranch", order = 0)}) + private int srvId; + + /** Suite name for queries */ + @AffinityKeyMapped + @QuerySqlField(orderedGroups = {@QuerySqlField.Group(name = "serverSuiteBranch", order = 1)}) + private int buildTypeId; + + /** Teamcity branch name for queries */ + @QuerySqlField(orderedGroups = {@QuerySqlField.Group(name = "serverSuiteBranch", order = 2)}) + private int normalizedBranchName; + + private Invocation suite; + + private Map tests = new HashMap<>(); + + Long buildStartTime; + + public SuiteInvocation() {} + + public SuiteInvocation(int srvId, int normalizedBaseBranch, FatBuildCompacted buildCompacted, IStringCompactor comp, + BiPredicate filter) { + this.srvId = srvId; + this.normalizedBranchName = normalizedBaseBranch; + this.buildStartTime = buildCompacted.getStartDateTs(); + this.suite = buildCompacted.toInvocation(comp, filter); + this.buildTypeId = buildCompacted.buildTypeId(); + } + + public void addTest(int testName, Invocation invocation) { + tests.put(testName, invocation); + } + + public Map tests() { + return Collections.unmodifiableMap(tests); + } + + public Invocation suiteInvocation() { + return suite; + } +} diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java new file mode 100644 index 000000000..dd93c5e8f --- /dev/null +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.tcignited.history; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import javax.cache.Cache; +import javax.cache.expiry.AccessedExpiryPolicy; +import javax.cache.expiry.Duration; +import javax.inject.Inject; +import javax.inject.Provider; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.QueryEntity; +import org.apache.ignite.cache.query.QueryCursor; +import org.apache.ignite.cache.query.SqlQuery; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; +import org.apache.ignite.tcbot.persistence.CacheConfigs; +import org.apache.ignite.tcignited.buildref.BuildRefDao; + +import static java.util.concurrent.TimeUnit.HOURS; + +/** + * Suite invocation history access object. + */ +public class SuiteInvocationHistoryDao { + /** Ignite provider. */ + @Inject + private Provider igniteProvider; + + /** Suite history cache. */ + private IgniteCache suiteHistory; + + public void init() { + CacheConfiguration ccfg = CacheConfigs.getCacheV2Config("teamcitySuiteHistory"); + ccfg.setExpiryPolicyFactory(AccessedExpiryPolicy.factoryOf(new Duration(HOURS, 12))); + ccfg.setEagerTtl(true); + + ccfg.setQueryEntities(Collections.singletonList(new QueryEntity(Long.class, SuiteInvocation.class))); + + Ignite ignite = igniteProvider.get(); + + suiteHistory = ignite.getOrCreateCache(ccfg); + } + + @AutoProfiling + public Map getSuiteRunHist(int srvId, int buildTypeId, int normalizedBranchName) { + java.util.Map map = new HashMap<>(); + try (QueryCursor> qryCursor = suiteHistory.query( + new SqlQuery(SuiteInvocation.class, "srvId = ? and buildTypeId = ? and normalizedBranchName = ?") + .setArgs(srvId, buildTypeId, normalizedBranchName))) { + + for (Cache.Entry next : qryCursor) { + Long key = next.getKey(); + int buildId = BuildRefDao.cacheKeyToBuildId(key); + map.put(buildId, next.getValue()); + } + } + + return map; + } + + @AutoProfiling + public void putAll(int srvId, Map addl) { + Map data = new HashMap<>(); + + addl.forEach((k, v) -> data.put(BuildRefDao.buildIdToCacheKey(srvId, k), v)); + + suiteHistory.putAll(data); + } +} diff --git a/tcbot-teamcity/build.gradle b/tcbot-teamcity/build.gradle index 4ed38063d..df805fb9c 100644 --- a/tcbot-teamcity/build.gradle +++ b/tcbot-teamcity/build.gradle @@ -16,17 +16,12 @@ */ apply plugin: 'java' - -repositories { - mavenCentral() - mavenLocal() -} + dependencies { compile (project(":tcbot-common")); - /// JAXB replacement for Java 11 - compile group: 'javax.xml.bind', name: 'jaxb-api', version: '2.3.1' + /// JAXB replacement for Java 11, API is defined in common compile group: 'com.sun.xml.bind', name: 'jaxb-impl', version: '2.3.0' compile group: 'com.sun.xml.bind', name: 'jaxb-core', version: '2.3.0' }