# -*- python -*-
# ex: set syntax=python:

from buildbot.buildslave import BuildSlave
from buildbot.changes.pb import PBChangeSource
from buildbot.scheduler import AnyBranchScheduler, Triggerable
from buildbot.schedulers.forcesched import FixedParameter, ForceScheduler, StringParameter, BooleanParameter
from buildbot.schedulers.filter import ChangeFilter
from buildbot.status import html
from buildbot.status.web.authz import Authz
from buildbot.process import buildstep, factory, properties
from buildbot.steps import master, shell, source, transfer, trigger
from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED

from twisted.internet import defer

import os
import re
import json
import operator
import cStringIO
import urllib

from committer_auth import CommitterAuth
import wkbuild


c = BuildmasterConfig = {}

c['change_source'] = PBChangeSource()

# permissions for WebStatus
authz = Authz(
    auth=CommitterAuth('auth.json'),
    forceBuild='auth',
    forceAllBuilds='auth',
    pingBuilder=True,
    gracefulShutdown=False,
    stopBuild='auth',
    stopAllBuilds='auth',
    cancelPendingBuild='auth',
    stopChange=True,
    cleanShutdown=False)

c['status'] = []
c['status'].append(html.WebStatus(http_port=8710,
                                  revlink="https://trac.webkit.org/changeset/%s", 
                                  changecommentlink=(r"(https://bugs\.webkit\.org/show_bug\.cgi\?id=|webkit\.org/b/)(\d+)", r"https://bugs.webkit.org/show_bug.cgi?id=\2"),
                                  authz=authz))

c['slavePortnum'] = 17000
c['projectName'] = "WebKit"
c['projectURL'] = "https://webkit.org"
c['buildbotURL'] = "https://build.webkit.org/"

c['buildHorizon'] = 1000
c['logHorizon'] = 500
c['eventHorizon'] = 200
c['buildCacheSize'] = 60

WithProperties = properties.WithProperties


class TestWithFailureCount(shell.Test):
    failedTestsFormatString = "%d test%s failed"

    def countFailures(self, cmd):
        return 0

    def commandComplete(self, cmd):
        shell.Test.commandComplete(self, cmd)
        self.failedTestCount = self.countFailures(cmd)
        self.failedTestPluralSuffix = "" if self.failedTestCount == 1 else "s"

    def evaluateCommand(self, cmd):
        if self.failedTestCount:
            return FAILURE

        if cmd.rc != 0:
            return FAILURE

        return SUCCESS

    def getText(self, cmd, results):
        return self.getText2(cmd, results)

    def getText2(self, cmd, results):
        if results != SUCCESS and self.failedTestCount:
            return [self.failedTestsFormatString % (self.failedTestCount, self.failedTestPluralSuffix)]

        return [self.name]


class ConfigureBuild(buildstep.BuildStep):
    name = "configure build"
    description = ["configuring build"]
    descriptionDone = ["configured build"]
    def __init__(self, platform, configuration, architecture, buildOnly, additionalArguments, SVNMirror, *args, **kwargs):
        buildstep.BuildStep.__init__(self, *args, **kwargs)
        self.platform = platform.split('-', 1)[0]
        self.fullPlatform = platform
        self.configuration = configuration
        self.architecture = architecture
        self.buildOnly = buildOnly
        self.additionalArguments = additionalArguments
        self.SVNMirror = SVNMirror
        self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly, additionalArguments=additionalArguments, SVNMirror=SVNMirror)

    def start(self):
        self.setProperty("platform", self.platform)
        self.setProperty("fullPlatform", self.fullPlatform)
        self.setProperty("configuration", self.configuration)
        self.setProperty("architecture", self.architecture)
        self.setProperty("buildOnly", self.buildOnly)
        self.setProperty("additionalArguments", self.additionalArguments)
        self.setProperty("SVNMirror", self.SVNMirror)
        self.finished(SUCCESS)
        return defer.succeed(None)


class CheckOutSource(source.SVN):
    mode = "update"
    def __init__(self, SVNMirror, **kwargs):
        kwargs['baseURL'] = SVNMirror or "http://svn.webkit.org/repository/webkit/"
        kwargs['defaultBranch'] = "trunk"
        kwargs['mode'] = self.mode
        source.SVN.__init__(self, **kwargs)
        self.addFactoryArguments(SVNMirror=SVNMirror)

class WaitForSVNServer(shell.ShellCommand):
    name = "wait-for-svn-server"
    command = ["python", "./Tools/BuildSlaveSupport/wait-for-SVN-server.py", "-r", WithProperties("%(revision)s"), "-s", WithProperties("%(SVNMirror)s")]
    description = ["waiting for SVN server"]
    descriptionDone = ["SVN server is ready"]
    warnOnFailure = True

    def evaluateCommand(self, cmd):
        if cmd.rc != 0:
            return WARNINGS
        return SUCCESS

class InstallWin32Dependencies(shell.Compile):
    description = ["installing dependencies"]
    descriptionDone = ["installed dependencies"]
    command = ["perl", "./Tools/Scripts/update-webkit-auxiliary-libs"]

class KillOldProcesses(shell.Compile):
    name = "kill old processes"
    description = ["killing old processes"]
    descriptionDone = ["killed old processes"]
    command = ["python", "./Tools/BuildSlaveSupport/kill-old-processes"]

class CleanBuildIfScheduled(shell.Compile):
    name = "delete WebKitBuild directory"
    description = ["deleting WebKitBuild directory"]
    descriptionDone = ["deleted WebKitBuild directory"]
    command = ["python", "./Tools/BuildSlaveSupport/clean-build", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]

    def start(self):
        if not self.getProperty('is_clean'):
            self.hideStepIf = True
            return SKIPPED
        return shell.Compile.start(self)

class DeleteStaleBuildFiles(shell.Compile):
    name = "delete stale build files"
    description = ["deleting stale build files"]
    descriptionDone = ["deleted stale build files"]
    command = ["python", "./Tools/BuildSlaveSupport/delete-stale-build-files", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]

    def start(self):
        if self.getProperty('is_clean'): # Nothing to be done if WebKitBuild had been removed.
            self.hideStepIf = True
            return SKIPPED
        return shell.Compile.start(self)

class InstallEflDependencies(shell.ShellCommand):
    name = "jhbuild"
    description = ["updating efl dependencies"]
    descriptionDone = ["updated efl dependencies"]
    command = ["perl", "./Tools/Scripts/update-webkitefl-libs"]
    haltOnFailure = True

class InstallGtkDependencies(shell.ShellCommand):
    name = "jhbuild"
    description = ["updating gtk dependencies"]
    descriptionDone = ["updated gtk dependencies"]
    command = ["perl", "./Tools/Scripts/update-webkitgtk-libs"]
    haltOnFailure = True

def appendCustomBuildFlags(step, platform, fullPlatform=""): 
    if platform in ('efl', 'gtk', 'wincairo'):
        step.setCommand(step.command + ['--' + platform])

class CompileWebKit(shell.Compile):
    command = ["perl", "./Tools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
    env = {'MFLAGS':''}
    name = "compile-webkit"
    description = ["compiling"]
    descriptionDone = ["compiled"]
    warningPattern = ".*arning: .*"

    def start(self):
        platform = self.getProperty('platform')
        buildOnly = self.getProperty('buildOnly')
        architecture = self.getProperty('architecture')
        if platform == 'mac' and architecture == 'i386':
            self.setCommand(self.command + ['--32-bit'])
        if platform == 'mac' and buildOnly:
            self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])

        appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))

        return shell.Compile.start(self)

    def createSummary(self, log):
        platform = self.getProperty('platform')
        if platform.startswith('mac'):    
            warnings = []
            errors = []
            sio = cStringIO.StringIO(log.getText())
            for line in sio.readlines():
                if "arning:" in line:
                    warnings.append(line)
                if "rror:" in line:
                    errors.append(line)
            if warnings:
                self.addCompleteLog('warnings', "".join(warnings))
            if errors:
                self.addCompleteLog('errors', "".join(errors))


class CompileLLINTCLoop(CompileWebKit):
    command = ["perl", "./Tools/Scripts/build-jsc", "--cloop", WithProperties("--%(configuration)s")]

class Compile32bitJSC(CompileWebKit):
    command = ["perl", "./Tools/Scripts/build-jsc", "--32-bit", WithProperties("--%(configuration)s")]


class ArchiveBuiltProduct(shell.ShellCommand):
    command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
               WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "archive"]
    name = "archive-built-product"
    description = ["archiving built product"]
    descriptionDone = ["archived built product"]
    haltOnFailure = True


class ExtractBuiltProduct(shell.ShellCommand):
    command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
               WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "extract"]
    name = "extract-built-product"
    description = ["extracting built product"]
    descriptionDone = ["extracted built product"]
    haltOnFailure = True


class UploadBuiltProduct(transfer.FileUpload):
    slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip")
    masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
    haltOnFailure = True

    def __init__(self, **kwargs):
        kwargs['slavesrc'] = self.slavesrc
        kwargs['masterdest'] = self.masterdest
        kwargs['mode'] = 0644
        kwargs['blocksize'] = 1024*256
        transfer.FileUpload.__init__(self, **kwargs)


class DownloadBuiltProduct(shell.ShellCommand):
    command = ["python", "./Tools/BuildSlaveSupport/download-built-product",
        WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"),
        WithProperties(c["buildbotURL"] + "archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")]
    name = "download-built-product"
    description = ["downloading built product"]
    descriptionDone = ["downloaded built product"]
    haltOnFailure = True
    flunkOnFailure = True


class RunJavaScriptCoreTests(TestWithFailureCount):
    name = "jscore-test"
    description = ["jscore-tests running"]
    descriptionDone = ["jscore-tests"]
    command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", "--no-build", WithProperties("--%(configuration)s")]
    failedTestsFormatString = "%d JSC test%s failed"

    def start(self):
        appendCustomBuildFlags(self, self.getProperty('platform'))
        return shell.Test.start(self)

    def countFailures(self, cmd):
        logText = cmd.logs['stdio'].getText()

        match = re.search(r'^Results for JSC stress tests:\r?\n\s+(\d+) failure', logText, re.MULTILINE)
        if match:
            return int(match.group(1))

        match = re.search(r'^Results for Mozilla tests:\r?\n\s+(\d+) regression', logText, re.MULTILINE)
        if match:
            return int(match.group(1))

        return 0


class RunRemoteJavaScriptCoreTests(RunJavaScriptCoreTests):
    def start(self):
        self.setCommand(self.command + ["--remote-config-file", "../../remote-jsc-tests-config.json"])
        return RunJavaScriptCoreTests.start(self)


class RunWebKitTests(shell.Test):
    name = "layout-test"
    description = ["layout-tests running"]
    descriptionDone = ["layout-tests"]
    command = ["python", "./Tools/Scripts/run-webkit-tests",
               "--no-build",
               "--no-show-results",
               "--no-new-test-results",
               "--no-sample-on-timeout",
               "--results-directory", "layout-test-results",
               "--builder-name", WithProperties("%(buildername)s"),
               "--build-number", WithProperties("%(buildnumber)s"),
               "--master-name", "webkit.org",
               "--test-results-server", "webkit-test-results.appspot.com",
               "--exit-after-n-crashes-or-timeouts", "50",
               "--exit-after-n-failures", "500",
               WithProperties("--%(configuration)s")]

    def start(self):
        platform = self.getProperty('platform')
        appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
        additionalArguments = self.getProperty('additionalArguments')

        if platform != "efl":
            self.setCommand(self.command + ['--debug-rwt-logging'])

        if platform == "win":
            self.setCommand(self.command + ['--batch-size', '100', '--root=' + os.path.join("WebKitBuild", self.getProperty('configuration'), "bin32")])

        if additionalArguments:
            self.setCommand(self.command + additionalArguments)
        return shell.Test.start(self)

    # FIXME: This will break if run-webkit-tests changes its default log formatter.
    nrwt_log_message_regexp = re.compile(r'\d{2}:\d{2}:\d{2}(\.\d+)?\s+\d+\s+(?P<message>.*)')

    def _strip_python_logging_prefix(self, line):
        match_object = self.nrwt_log_message_regexp.match(line)
        if match_object:
            return match_object.group('message')
        return line

    def _parseRunWebKitTestsOutput(self, logText):
        incorrectLayoutLines = []
        expressions = [
            ('flakes', re.compile(r'Unexpected flakiness.+\((\d+)\)')),
            ('new passes', re.compile(r'Expected to .+, but passed:\s+\((\d+)\)')),
            ('missing results', re.compile(r'Regressions: Unexpected missing results\s+\((\d+)\)')),
            ('failures', re.compile(r'Regressions: Unexpected.+\((\d+)\)')),
        ]
        testFailures = {}

        for line in logText.splitlines():
            if line.find('Exiting early') >= 0 or line.find('leaks found') >= 0:
                incorrectLayoutLines.append(self._strip_python_logging_prefix(line))
                continue
            for name, expression in expressions:
                match = expression.search(line)

                if match:
                    testFailures[name] = testFailures.get(name, 0) + int(match.group(1))
                    break

                # FIXME: Parse file names and put them in results

        for name in testFailures:
            incorrectLayoutLines.append(str(testFailures[name]) + ' ' + name)

        self.incorrectLayoutLines = incorrectLayoutLines

    def commandComplete(self, cmd):
        shell.Test.commandComplete(self, cmd)

        logText = cmd.logs['stdio'].getText()
        self._parseRunWebKitTestsOutput(logText)

    def evaluateCommand(self, cmd):
        result = SUCCESS

        if self.incorrectLayoutLines:
            if len(self.incorrectLayoutLines) == 1:
                line = self.incorrectLayoutLines[0]
                if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:
                    return WARNINGS

            for line in self.incorrectLayoutLines:
                if line.find('flakes') >= 0 or line.find('new passes') >= 0 or line.find('missing results') >= 0:
                    result = WARNINGS
                else:
                    return FAILURE

        if cmd.rc != 0:
            return FAILURE

        return result

    def getText(self, cmd, results):
        return self.getText2(cmd, results)

    def getText2(self, cmd, results):
        if results != SUCCESS and self.incorrectLayoutLines:
            return self.incorrectLayoutLines

        return [self.name]


class RunUnitTests(TestWithFailureCount):
    name = "run-api-tests"
    description = ["unit tests running"]
    descriptionDone = ["unit-tests"]
    command = ["perl", "./Tools/Scripts/run-api-tests", "--no-build", WithProperties("--%(configuration)s"), "--verbose"]
    failedTestsFormatString = "%d unit test%s failed or timed out"

    def countFailures(self, cmd):
        log_text = cmd.logs['stdio'].getText()
        count = 0

        split = re.split(r'\sTests that timed out:\s', log_text)
        if len(split) > 1:
            count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))

        split = re.split(r'\sTests that failed:\s', split[0])
        if len(split) > 1:
            count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))

        return count


class RunPythonTests(TestWithFailureCount):
    name = "webkitpy-test"
    description = ["python-tests running"]
    descriptionDone = ["python-tests"]
    command = ["python", "./Tools/Scripts/test-webkitpy", "--verbose"]
    failedTestsFormatString = "%d python test%s failed"

    def start(self):
        platform = self.getProperty('platform')
        # Python tests are flaky on the GTK builders, running them serially
        # helps and does not significantly prolong the cycle time.
        if platform == 'gtk':
            self.setCommand(self.command + ['--child-processes', '1'])
        # Python tests fail on windows bots when running more than one child process
        # https://bugs.webkit.org/show_bug.cgi?id=97465
        if platform == 'win':
            self.setCommand(self.command + ['--child-processes', '1'])
        return shell.Test.start(self)

    def countFailures(self, cmd):
        logText = cmd.logs['stdio'].getText()
        # We're looking for the line that looks like this: FAILED (failures=2, errors=1)
        regex = re.compile(r'^FAILED \((?P<counts>[^)]+)\)')
        for line in logText.splitlines():
            match = regex.match(line)
            if not match:
                continue
            return sum(int(component.split('=')[1]) for component in match.group('counts').split(', '))
        return 0


class RunPerlTests(TestWithFailureCount):
    name = "webkitperl-test"
    description = ["perl-tests running"]
    descriptionDone = ["perl-tests"]
    command = ["perl", "./Tools/Scripts/test-webkitperl"]
    failedTestsFormatString = "%d perl test%s failed"

    def countFailures(self, cmd):
        logText = cmd.logs['stdio'].getText()
        # We're looking for the line that looks like this: Failed 2/19 test programs. 5/363 subtests failed.
        regex = re.compile(r'^Failed \d+/\d+ test programs\. (?P<count>\d+)/\d+ subtests failed\.')
        for line in logText.splitlines():
            match = regex.match(line)
            if not match:
                continue
            return int(match.group('count'))
        return 0


class RunLLINTCLoopTests(TestWithFailureCount):
    name = "webkit-jsc-cloop-test"
    description = ["cloop-tests running"]
    descriptionDone = ["cloop-tests"]
    command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", "--cloop", "--no-build", "--no-jsc-stress", WithProperties("--%(configuration)s")]
    failedTestsFormatString = "%d regression%s found."

    def countFailures(self, cmd):
        logText = cmd.logs['stdio'].getText()
        # We're looking for the line that looks like this: 0 regressions found.
        regex = re.compile(r'\s*(?P<count>\d+) regressions? found.')
        for line in logText.splitlines():
            match = regex.match(line)
            if not match:
                continue
            return int(match.group('count'))
        return 0


class Run32bitJSCTests(TestWithFailureCount):
    name = "webkit-32bit-jsc-test"
    description = ["32bit-jsc-tests running"]
    descriptionDone = ["32bit-jsc-tests"]
    command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", "--32-bit", "--no-build", WithProperties("--%(configuration)s")]
    failedTestsFormatString = "%d regression%s found."

    def countFailures(self, cmd):
        logText = cmd.logs['stdio'].getText()
        # We're looking for the line that looks like this: 0 failures found.
        regex = re.compile(r'\s*(?P<count>\d+) failures? found.')
        for line in logText.splitlines():
            match = regex.match(line)
            if not match:
                continue
            return int(match.group('count'))
        return 0


class RunBindingsTests(shell.Test):
    name = "bindings-generation-tests"
    description = ["bindings-tests running"]
    descriptionDone = ["bindings-tests"]
    command = ["python", "./Tools/Scripts/run-bindings-tests"]


class RunEflAPITests(shell.Test):
    name = "API tests"
    description = ["API tests running"]
    descriptionDone = ["API tests"]
    command = ["perl", "./Tools/Scripts/run-efl-tests", WithProperties("--%(configuration)s")]


class RunGtkAPITests(shell.Test):
    name = "API tests"
    description = ["API tests running"]
    descriptionDone = ["API tests"]
    command = ["python", "./Tools/Scripts/run-gtk-tests", "--verbose", WithProperties("--%(configuration)s")]

    def commandComplete(self, cmd):
        shell.Test.commandComplete(self, cmd)

        logText = cmd.logs['stdio'].getText()

        self.incorrectTests = 0
        self.crashedTests = 0
        self.timedOutTests = 0
        self.skippedTests = 0
        self.statusLine = []

        foundItems = re.findall("Tests failed \((\d+)\):", logText)
        if (foundItems):
            self.incorrectTests = int(foundItems[0])

        foundItems = re.findall("Tests that crashed \((\d+)\):", logText)
        if (foundItems):
            self.crashedTests = int(foundItems[0])

        foundItems = re.findall("Tests that timed out \((\d+)\):", logText)
        if (foundItems):
            self.timedOutTests = int(foundItems[0])

        foundItems = re.findall("Tests skipped \((\d+)\):", logText)
        if (foundItems):
            self.skippedTests = int(foundItems[0])

        self.totalFailedTests = self.incorrectTests + self.crashedTests + self.timedOutTests

        if self.totalFailedTests > 0:
            self.statusLine = [
                "%d API tests failed, %d crashed, %d timed out, %d skipped" %
                (self.incorrectTests, self.crashedTests, self.timedOutTests, self.skippedTests)
            ]

    def evaluateCommand(self, cmd):
        if self.totalFailedTests > 0:
            return FAILURE

        if cmd.rc != 0:
            return FAILURE

        return SUCCESS

    def getText(self, cmd, results):
        return self.getText2(cmd, results)

    def getText2(self, cmd, results):
        if results != SUCCESS and self.totalFailedTests > 0:
            return self.statusLine

        return [self.name]

class RunGtkWebKitGObjectDOMBindingsAPIBreakTests(shell.Test):
    name = "WebKit GObject DOM bindings API break tests"
    description = ["WebKit GObject DOM bindings API break tests running"]
    descriptionDone = ["WebKit GObject DOM bindings API break tests"]
    command = ["./Tools/gtk/check-for-webkitdom-api-breaks"]

    def commandComplete(self, cmd):
        shell.Test.commandComplete(self, cmd)

        logText = cmd.logs['stdio'].getText()

        self.missingAPI = bool(re.findall("Missing API", logText))
        self.newAPI = bool(re.findall("New API", logText))

        line = ""
        if self.newAPI:
            line += "Found new API."
        if self.missingAPI:
            line += "Found missing API."
        if line:
            self.statusLine = [ line ]

    def evaluateCommand(self, cmd):
        if self.missingAPI:
            return FAILURE
        if self.newAPI:
            return WARNINGS
        if cmd.rc != 0:
            return FAILURE
        return SUCCESS

class RunWebKit1Tests(RunWebKitTests):
    def start(self):
        self.setCommand(self.command + ["--dump-render-tree"])

        return RunWebKitTests.start(self)

class RunWebKit1LeakTests(RunWebKit1Tests):
    warnOnWarnings = True
    def start(self):
        self.setCommand(self.command + ["--leaks"])
        return RunWebKit1Tests.start(self)

class RunAndUploadPerfTests(shell.Test):
    name = "perf-test"
    description = ["perf-tests running"]
    descriptionDone = ["perf-tests"]
    command = ["python", "./Tools/Scripts/run-perf-tests",
               "--output-json-path", "perf-test-results.json",
               "--slave-config-json-path", "../../perf-test-config.json",
               "--no-show-results",
               "--reset-results",
               "--test-results-server", "perf.webkit.org",
               "--builder-name", WithProperties("%(buildername)s"),
               "--build-number", WithProperties("%(buildnumber)s"),
               "--platform", WithProperties("%(fullPlatform)s"),
               "--no-build",
               WithProperties("--%(configuration)s")]

    def start(self):
        self.setCommand(self.command)
        return shell.Test.start(self)

    def getText(self, cmd, results):
        return self.getText2(cmd, results)

    def getText2(self, cmd, results):
        if results != SUCCESS:
            if cmd.rc == -1 & 0xff:
                return ["build not up to date"]
            elif cmd.rc == -2 & 0xff:
                return ["slave config JSON error"]
            elif cmd.rc == -3 & 0xff:
                return ["output JSON merge error"]
            elif cmd.rc == -4 & 0xff:
                return ["upload error"]
            elif cmd.rc == -5 & 0xff:
                return ["system dependency error"]
            elif cmd.rc == -1:
                return ["timeout"]
            else:
                return ["%d perf tests failed" % cmd.rc]

        return [self.name]


class ArchiveTestResults(shell.ShellCommand):
    command = ["python", "./Tools/BuildSlaveSupport/test-result-archive",
               WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
    name = "archive-test-results"
    description = ["archiving test results"]
    descriptionDone = ["archived test results"]
    haltOnFailure = True


class UploadTestResults(transfer.FileUpload):
    slavesrc = "layout-test-results.zip"
    masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")

    def __init__(self, **kwargs):
        kwargs['slavesrc'] = self.slavesrc
        kwargs['masterdest'] = self.masterdest
        kwargs['mode'] = 0644
        transfer.FileUpload.__init__(self, **kwargs)


class ExtractTestResults(master.MasterShellCommand):
    zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
    resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)")
    descriptionDone = ["uploaded results"]

    def __init__(self, **kwargs):
        kwargs['command'] = ""
        master.MasterShellCommand.__init__(self, **kwargs)

    def resultDirectoryURL(self):
        return self.build.getProperties().render(self.resultDirectory).replace("public_html/", "/") + "/"

    def start(self):
        self.command = ["unzip", self.build.getProperties().render(self.zipFile), "-d", self.build.getProperties().render(self.resultDirectory)]
        return master.MasterShellCommand.start(self)

    def addCustomURLs(self):
        url = self.resultDirectoryURL() + "results.html"
        self.addURL("view results", url)

    def finished(self, result):
        self.addCustomURLs()
        return master.MasterShellCommand.finished(self, result)


class ExtractTestResultsAndLeaks(ExtractTestResults):
    def addCustomURLs(self):
        ExtractTestResults.addCustomURLs(self)
        url = "/LeaksViewer/?url=" + urllib.quote(self.resultDirectoryURL(), safe="")
        self.addURL("view leaks", url)


class Factory(factory.BuildFactory):
    def __init__(self, platform, configuration, architectures, buildOnly, additionalArguments, SVNMirror):
        factory.BuildFactory.__init__(self)
        self.addStep(ConfigureBuild(platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly, additionalArguments=additionalArguments, SVNMirror=SVNMirror))
        if SVNMirror:
            self.addStep(WaitForSVNServer())
        self.addStep(CheckOutSource(SVNMirror=SVNMirror))
        if not (platform == "efl" and ('armv7' in architectures or 'aarch64' in architectures)):
            self.addStep(KillOldProcesses())
        self.addStep(CleanBuildIfScheduled())
        self.addStep(DeleteStaleBuildFiles())
        if platform == "win":
            self.addStep(InstallWin32Dependencies())
        if platform == "gtk":
            self.addStep(InstallGtkDependencies())
        if platform == "efl":
            self.addStep(InstallEflDependencies())


class BuildFactory(Factory):
    def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None):
        Factory.__init__(self, platform, configuration, architectures, True, additionalArguments, SVNMirror)

        if platform == "win":
            self.addStep(CompileWebKit(timeout=2*60*60))
        else:
            self.addStep(CompileWebKit())

        if triggers:
            self.addStep(ArchiveBuiltProduct())
            self.addStep(UploadBuiltProduct())
            self.addStep(trigger.Trigger(schedulerNames=triggers))

def pickLatestBuild(builder, requests):
    return max(requests, key=operator.attrgetter("submittedAt"))

class TestFactory(Factory):
    JSCTestClass = RunJavaScriptCoreTests
    LayoutTestClass = RunWebKitTests

    def getProduct(self):
        self.addStep(DownloadBuiltProduct())
        self.addStep(ExtractBuiltProduct())

    def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None, **kwargs):
        Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
        self.getProduct()
        if self.JSCTestClass:
            self.addStep(self.JSCTestClass())
        if self.LayoutTestClass:
            self.addStep(self.LayoutTestClass())

        if platform == 'win' or platform.startswith('mac'):
            self.addStep(RunUnitTests())
        self.addStep(RunPythonTests())
        self.addStep(RunPerlTests())
        self.addStep(RunBindingsTests())
        if self.LayoutTestClass:
            self.addStep(ArchiveTestResults())
            self.addStep(UploadTestResults())
            self.addStep(ExtractTestResults())
        if platform == "efl":
            self.addStep(RunEflAPITests())
        if platform == "gtk":
            self.addStep(RunGtkAPITests())
            self.addStep(RunGtkWebKitGObjectDOMBindingsAPIBreakTests())

class BuildAndTestFactory(TestFactory):
    def getProduct(self):
        self.addStep(CompileWebKit())

    def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None, **kwargs):
        TestFactory.__init__(self, platform, configuration, architectures, additionalArguments, SVNMirror, **kwargs)
        if triggers:
            self.addStep(ArchiveBuiltProduct())
            self.addStep(UploadBuiltProduct())
            self.addStep(trigger.Trigger(schedulerNames=triggers))

class BuildAndTestLLINTCLoopFactory(Factory):
    def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None, **kwargs):
        Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
        self.addStep(CompileLLINTCLoop())
        self.addStep(RunLLINTCLoopTests())

class BuildAndTest32bitJSCFactory(Factory):
    def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None, **kwargs):
        Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
        self.addStep(Compile32bitJSC())
        self.addStep(Run32bitJSCTests())

class BuildAndNonLayoutTestFactory(BuildAndTestFactory):
    LayoutTestClass = None

class BuildAndRemoteJSCTestsFactory(BuildFactory):
    def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None):
        BuildFactory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
        self.addStep(RunRemoteJavaScriptCoreTests())

class TestWebKit1LeaksFactory(Factory):
    def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None):
        Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
        self.addStep(DownloadBuiltProduct())
        self.addStep(ExtractBuiltProduct())
        self.addStep(RunWebKit1LeakTests())
        self.addStep(ArchiveTestResults())
        self.addStep(UploadTestResults())
        self.addStep(ExtractTestResultsAndLeaks())

class TestAllButJSCFactory(TestFactory):
    JSCTestClass = None

class TestJSCFactory(Factory):
    def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None):
        Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
        self.addStep(DownloadBuiltProduct())
        self.addStep(ExtractBuiltProduct())
        self.addStep(RunJavaScriptCoreTests())

class TestWebKit1Factory(TestFactory):
    LayoutTestClass = RunWebKit1Tests

class TestWebKit1AllButJSCFactory(TestWebKit1Factory):
    JSCTestClass = None

class BuildAndPerfTestFactory(Factory):
    def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None, **kwargs):
        Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
        self.addStep(CompileWebKit())
        self.addStep(RunAndUploadPerfTests())

class DownloadAndPerfTestFactory(Factory):
    def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None, **kwargs):
        Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
        self.addStep(DownloadBuiltProduct())
        self.addStep(ExtractBuiltProduct())
        self.addStep(RunAndUploadPerfTests())

class PlatformSpecificScheduler(AnyBranchScheduler):
    def __init__(self, platform, branch, **kwargs):
        self.platform = platform
        filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter)
        AnyBranchScheduler.__init__(self, name=platform, change_filter=filter, **kwargs)

    def filter(self, change):
        return wkbuild.should_build(self.platform, change.files)

trunk_filter = ChangeFilter(branch=["trunk", None])

def loadBuilderConfig(c):
    # FIXME: These file handles are leaked.
    passwords = json.load(open('passwords.json'))
    config = json.load(open('config.json'))

    c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']]

    c['schedulers'] = []
    for scheduler in config['schedulers']:
        if "change_filter" in scheduler:
            scheduler["change_filter"] = globals()[scheduler["change_filter"]]
        kls = globals()[scheduler.pop('type')]
        # Python 2.6 can't handle unicode keys as keyword arguments:
        # http://bugs.python.org/issue2646.  Modern versions of json return
        # unicode strings from json.load, so we map all keys to str objects.
        scheduler = dict(map(lambda key_value_pair: (str(key_value_pair[0]), key_value_pair[1]), scheduler.items()))

        c['schedulers'].append(kls(**scheduler))

    forceScheduler = ForceScheduler(
        name="force",
        builderNames=[str(builder['name']) for builder in config['builders']],
        reason=StringParameter(name="reason", default="", size=40),

        # Validate SVN revision: number or empty string
        revision=StringParameter(name="revision", default="", regex=re.compile(r'^(\d*)$')),

        # Disable default enabled input fields: branch, repository, project, additional properties
        branch=FixedParameter(name="branch"),
        repository=FixedParameter(name="repository"),
        project=FixedParameter(name="project"),
        properties=[BooleanParameter(name="is_clean", label="Force Clean build")]
    )
    c['schedulers'].append(forceScheduler)

    c['builders'] = []
    for builder in config['builders']:
        for slaveName in builder['slavenames']:
            for slave in config['slaves']:
                if slave['name'] != slaveName or slave['platform'] == '*':
                    continue

                if slave['platform'] != builder['platform']:
                    raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform'])

                break

        platform = builder['platform']

        builderType = builder.pop('type')
        factory = globals()["%sFactory" % builderType]
        factorykwargs = {}
        for key in "platform", "configuration", "architectures", "triggers", "additionalArguments", "SVNMirror":
            value = builder.pop(key, None)
            if value:
                factorykwargs[key] = value

        builder["factory"] = factory(**factorykwargs)

        if platform.startswith('mac'):
            builder["category"] = 'AppleMac'
        elif platform == 'win':
            builder["category"] = 'AppleWin'
        elif platform.startswith('gtk'):
            builder["category"] = 'GTK'
        elif platform.startswith('efl'):
            builder["category"] = "EFL"
        else:
            builder["category"] = 'misc'

        if (builder['category'] == 'AppleMac' or builder['category'] == 'AppleWin') and builderType != 'Build':
            builder['nextBuild'] = pickLatestBuild

        c['builders'].append(builder)

loadBuilderConfig(c)