diff --git a/backend/ec2.go b/backend/ec2.go new file mode 100644 index 000000000..49303a438 --- /dev/null +++ b/backend/ec2.go @@ -0,0 +1,742 @@ +package backend + +import ( + "bytes" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "fmt" + "io" + "net" + "net/url" + "regexp" + "strconv" + "strings" + "text/template" + "time" + + gocontext "context" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + "github.com/travis-ci/worker/config" + "github.com/travis-ci/worker/context" + "github.com/travis-ci/worker/image" + "github.com/travis-ci/worker/ssh" +) + +var ( + defaultEC2SSHDialTimeout = 5 * time.Second + defaultEC2ImageSelectorType = "env" + defaultEC2SSHUserName = "travis" + defaultEC2ExecCmd = "bash ~/build.sh" + defaultEC2SubnetID = "" + defaultEC2InstanceType = "t2.micro" + defaultEC2Image = "ami-02790d1ebf3b5181d" + defaultEC2SecurityGroupIDs = "default" + defaultEC2EBSOptimized = false + defaultEC2DiskSize = int64(8) + defaultEC2UploadRetries = uint64(120) + defaultEC2UploadRetrySleep = 1 * time.Second + defaultEC2Region = "eu-west-1" +) + +var ( + ec2StartupScript = template.Must(template.New("ec2-startup").Parse(`#!/usr/bin/env bash + +cat > ~travis/.ssh/authorized_keys < p.uploadRetries { + instanceChan <- nil + return + } + time.Sleep(500 * time.Millisecond) + } + }() + + select { + case instance := <-instanceChan: + if instance != nil { + return &ec2Instance{ + provider: p, + sshDialer: sshDialer, + endBooting: time.Now(), + startBooting: startBooting, + instance: instance, + tmpKeyName: keyResp.KeyName, + }, nil + } + return nil, lastErr + case <-ctx.Done(): + context.LoggerFromContext(ctx).WithFields(logrus.Fields{ + "err": lastErr, + "self": "backend/ec2_instance", + }).Info("Stopping probing for up instance") + return nil, ctx.Err() + } +} + +func (p *ec2Provider) Setup(ctx gocontext.Context) error { + return nil +} + +type ec2Instance struct { + provider *ec2Provider + startBooting time.Time + endBooting time.Time + sshDialer ssh.Dialer + instance *ec2.Instance + tmpKeyName *string +} + +func (i *ec2Instance) UploadScript(ctx gocontext.Context, script []byte) error { + defer context.TimeSince(ctx, "boot_poll_ssh", time.Now()) + uploadedChan := make(chan error) + var lastErr error + + logger := context.LoggerFromContext(ctx).WithField("self", "backend/ec2_instance") + + // Wait for ssh to becom available + go func() { + var errCount uint64 + for { + if ctx.Err() != nil { + return + } + + err := i.uploadScriptAttempt(ctx, script) + if err != nil { + logger.WithError(err).Debug("upload script attempt errored") + } else { + uploadedChan <- nil + return + } + + lastErr = err + + errCount++ + if errCount > i.provider.uploadRetries { + uploadedChan <- err + return + } + time.Sleep(i.provider.uploadRetrySleep) + } + }() + + select { + case err := <-uploadedChan: + return err + case <-ctx.Done(): + context.LoggerFromContext(ctx).WithFields(logrus.Fields{ + "err": lastErr, + "self": "backend/ec2_instance", + }).Info("stopping upload retries, error from last attempt") + return ctx.Err() + } + + //return i.uploadScriptAttempt(ctx, script) +} + +func (i *ec2Instance) waitForSSH(port, timeout int) error { + + host := *i.instance.PrivateIpAddress + if i.provider.publicIPConnect { + host = *i.instance.PublicIpAddress + } + + iter := 0 + for { + _, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", host, port), 1*time.Second) + if err == nil { + break + } + iter = iter + 1 + if iter > timeout { + return err + } + time.Sleep(500 * time.Millisecond) + } + return nil +} + +func (i *ec2Instance) uploadScriptAttempt(ctx gocontext.Context, script []byte) error { + return i.uploadScriptSCP(ctx, script) +} + +func (i *ec2Instance) uploadScriptSCP(ctx gocontext.Context, script []byte) error { + conn, err := i.sshConnection(ctx) + if err != nil { + return err + } + defer conn.Close() + + existed, err := conn.UploadFile("build.sh", script) + if existed { + return ErrStaleVM + } + if err != nil { + return errors.Wrap(err, "couldn't upload build script") + } + return nil +} + +func (i *ec2Instance) sshConnection(ctx gocontext.Context) (ssh.Connection, error) { + ip := *i.instance.PrivateIpAddress + if i.provider.publicIPConnect { + ip = *i.instance.PublicIpAddress + } + return i.sshDialer.Dial(fmt.Sprintf("%s:22", ip), defaultEC2SSHUserName, i.provider.sshDialTimeout) +} + +func (i *ec2Instance) RunScript(ctx gocontext.Context, output io.Writer) (*RunResult, error) { + return i.runScriptSSH(ctx, output) +} + +func (i *ec2Instance) runScriptSSH(ctx gocontext.Context, output io.Writer) (*RunResult, error) { + conn, err := i.sshConnection(ctx) + if err != nil { + return &RunResult{Completed: false}, errors.Wrap(err, "couldn't connect to SSH server") + } + defer conn.Close() + + exitStatus, err := conn.RunCommand(strings.Join(i.provider.execCmd, " "), output) + + return &RunResult{Completed: err != nil, ExitCode: exitStatus}, errors.Wrap(err, "error running script") +} + +func (i *ec2Instance) Stop(ctx gocontext.Context) error { + logger := context.LoggerFromContext(ctx).WithField("self", "backend/ec2_provider") + //hostName := hostnameFromContext(ctx) + + svc := ec2.New(i.provider.awsSession) + + instanceTerminationInput := &ec2.TerminateInstancesInput{ + InstanceIds: []*string{ + i.instance.InstanceId, + }, + } + + _, err := svc.TerminateInstances(instanceTerminationInput) + + if err != nil { + return err + } + + logger.Info(fmt.Sprintf("Terminated instance %s with hostname %s", *i.instance.InstanceId, *i.instance.PrivateDnsName)) + + deleteKeyPairInput := &ec2.DeleteKeyPairInput{ + KeyName: i.tmpKeyName, + } + + _, err = svc.DeleteKeyPair(deleteKeyPairInput) + + if err != nil { + return err + } + + logger.Info(fmt.Sprintf("Deleted keypair %s", *i.tmpKeyName)) + + return nil +} + +func (i *ec2Instance) DownloadTrace(gocontext.Context) ([]byte, error) { + return nil, nil +} + +func (i *ec2Instance) SupportsProgress() bool { + return false +} + +func (i *ec2Instance) Warmed() bool { + return false +} + +func (i *ec2Instance) ID() string { + if i.provider.publicIP { + return *i.instance.PublicDnsName + } + return *i.instance.PrivateDnsName +} + +func (i *ec2Instance) ImageName() string { + return "ec2" +} + +func (i *ec2Instance) StartupDuration() time.Duration { + return i.endBooting.Sub(i.startBooting) +} diff --git a/vendor/manifest b/vendor/manifest index 5b22f7598..4fe2dfd84 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -282,6 +282,15 @@ "path": "service/sts", "notests": true }, + { + "importpath": "github.com/aws/aws-sdk-go/service/ec2", + "repository": "https://github.com/aws/aws-sdk-go", + "vcs": "git", + "revision": "c15e3069f617b6020f1ada3eee769a61318dabdf", + "branch": "master", + "path": "/service/ec2", + "notests": true + }, { "importpath": "github.com/aws/aws-sdk-go/vendor/github.com/go-ini/ini", "repository": "https://github.com/aws/aws-sdk-go", @@ -1581,4 +1590,4 @@ "notests": true } ] -} \ No newline at end of file +}