Issue #192 Making sure Proxy retrieves cluster view from Idler

- Removing obsolete config options
- Updating README and adding info about how to trigger webhooks
- Some logging improvements

README.md

@@ -14,11 +14,13 @@
     - [Check commit message format](#check-commit-message-format)
     - [Clean up](#clean-up)
   - [Dependency management](#dependency-management)
+- [Continuous Integration](#continuous-integration)
+- [Running locally](#running-locally)
+  - [Testing webhooks](#testing-webhooks)
 
 <!-- /MarkdownTOC -->
 
-
-<a name="what-is-it"></a>
+<a id="what-is-it"></a>
 # What is it?
 
 fabric8-jenkins-proxy (Jenkins Proxy) is the sister project to [fabric8-jenkins-idler](https://github.com/fabric8-services/fabric8-jenkins-idler)(Jenkins Idler).
@@ -27,7 +29,7 @@ For more information refer to the Idler [README](https://github.com/fabric8-serv
 
 ![Architectural Diagram](https://camo.githubusercontent.com/0761536bd1260ce502604e4d2ff2592a79f56485/68747470733a2f2f646f63732e676f6f676c652e636f6d2f64726177696e67732f642f652f32504143582d31765268743172674e45533636663732395155634e356f475378745453475667554c5f38725f632d4b5f4a722d694b304657654844616b354933326c31794d69592d744e2d6e715168495259766f31472f7075623f773d34323626683d343431)
 
-<a name="data-flow-diagrams"></a>
+<a id="data-flow-diagrams"></a>
 # Data flow diagrams
 
 The following diagrams describe the data flow within the proxy for a received GitHub webhook respectively a direct user interaction with the Jenkins service:
@@ -36,12 +38,12 @@ The following diagrams describe the data flow within the proxy for a received Gi
 
 [![Jenkins UI](./docs/jenkins-ui.png)](./docs/jenkins-ui.png)
 
-<a name="how-to-build"></a>
+<a id="how-to-build"></a>
 # How to build?
 
 The following paragraphs describe how to build and work with the source.
 
-<a name="prerequisites"></a>
+<a id="prerequisites"></a>
 ## Prerequisites
 
 The project is written in [Go](https://golang.org/), so you will need a working Go installation (Go version >= 1.9.1).
@@ -50,42 +52,42 @@ The build itself is driven by GNU [Make](https://www.gnu.org/software/make/) whi
 
 Last but not least, you need a running Docker daemon, since the final build artifact is a Docker container.
 
-<a name="make-usage"></a>
+<a id="make-usage"></a>
 ## Make usage
 
-<a name="compile-the-code"></a>
+<a id="compile-the-code"></a>
 ### Compile the code
 
    $ make build
 
-<a name="build-the-container-image"></a>
+<a id="build-the-container-image"></a>
 ### Build the container image
 
    $ make image
 
-<a name="run-the-tests"></a>
+<a id="run-the-tests"></a>
 ### Run the tests
 
    $ make test
 
-<a name="format-the-code"></a>
+<a id="format-the-code"></a>
 ### Format the code
 
    $ make fmt
 
-<a name="check-commit-message-format"></a>
+<a id="check-commit-message-format"></a>
 ### Check commit message format
 
    $ make validate_commits
 
-<a name="clean-up"></a>
+<a id="clean-up"></a>
 ### Clean up
 
    $ make clean
 
 More help is provided by `make help`.
 
-<a name="dependency-management"></a>
+<a id="dependency-management"></a>
 ## Dependency management
 
 The dependencies of the project are managed by [Dep](https://github.com/golang/dep).
@@ -99,7 +101,40 @@ The process looks like this:
     $ git add Gopkg.toml Gopkg.lock
     $ git commit
 
+<a id="continuous-integration"></a>
+# Continuous Integration
+
+At the moment Travis CI and CentOS CI are configured.
+Both CI systems build all merges to master as well as pull requests.
+
+| CI System |   |
+|-----------|---|
+| CentOS CI | [master](https://ci.centos.org/job/devtools-fabric8-jenkins-proxy-build-master/), [pr](https://ci.centos.org/job/devtools-fabric8-jenkins-proxy/)|
+| Travis CI | [master](https://travis-ci.org/fabric8-services/fabric8-jenkins-proxy/), [pr](https://travis-ci.org/fabric8-services/fabric8-jenkins-proxy/pull_requests)|
+
+<a id="running-locally"></a>
+# Running locally
+
+The repository contains a script [`setupLocalProxy.sh`](./scripts/setupLocalProxy.sh) which can be used to run the Proxy locally.
+A prerequisite for this is access to https://console.rh-idev.openshift.com/.
+To run the script you need to export your OpenShift access token for console.rh-idev.openshift.com as DSAAS_PREVIEW_TOKEN.
+Note, you need edit permissions on the dsaas-preview namespace in order to port forward.
+
+<a id="testing-webhooks"></a>
+## Testing webhooks
+
+You can trigger local webhook delivery like so.
+Go to a GitHub repository generated by the OpenShift.io launcher.
+Find the webhook settings under Settings->Webhooks.
+There you can see the recent deliveries.
+Copy the payload of a webhook delivery into a file `webhook-payload.json`.
+Then execute the following curl command:
 
+    $ curl http://localhost:8080/github-webhook/ \
+    -H "Content-Type: application/json" \
+    -H "User-Agent: GitHub-Hookshot/c494ff1" \
+    -H "X-GitHub-Event: status" \
+    -d @webhook-payload.json
 
 
 

cmd/fabric8-jenkins-proxy/main.go

@@ -56,15 +56,15 @@ func init() {
 func main() {
 	mainLogger.Infof("Proxy version: %s", version.GetVersion())
 
-	//Init configuration
+	// Init configuration
 	config, err := configuration.NewConfiguration()
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	mainLogger.Infof("Proxy config: %s", config.String())
 
-	//Connect to DB
+	// Connect to DB
 	db, err := storage.Connect(config)
 	if err != nil {
 		log.Fatal(err)
@@ -73,26 +73,32 @@ func main() {
 
 	store := storage.NewDBStorage(db)
 
-	//Create tenant client
+	// Create tenant client
 	tenant := clients.NewTenant(config.GetTenantURL(), config.GetAuthToken())
 
-	//Create WorkItemTracker client
+	// Create WorkItemTracker client
 	wit := clients.NewWIT(config.GetWitURL(), config.GetAuthToken())
 
-	//Create Idler client
+	// Create Idler client
 	idler := clients.NewIdler(config.GetIdlerURL())
 
-	start(config, &tenant, &wit, idler, store)
+	// Get the cluster view from the Idler
+	clusters, err := idler.Clusters()
+	if err != nil {
+		mainLogger.WithField("error", err).Fatalf("Failure to retrieve cluster view")
+	}
+	mainLogger.WithField("clusters", clusters).Info("Retrieved cluster view")
+
+	start(config, &tenant, &wit, idler, store, clusters)
 }
 
-func start(config configuration.Configuration, tenant *clients.Tenant, wit *clients.WIT, idler clients.IdlerService, store storage.Store) {
-	proxy, err := proxy.NewProxy(tenant, wit, idler, store, config)
+func start(config configuration.Configuration, tenant *clients.Tenant, wit *clients.WIT, idler clients.IdlerService, store storage.Store, clusters map[string]string) {
+	proxy, err := proxy.NewProxy(tenant, wit, idler, store, config, clusters)
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	// Start the various Go routines
-	// TODO - Eventually all goroutines should be started and controlled from the method below
 	var wg sync.WaitGroup
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()

cmd/fabric8-jenkins-proxy/main_test.go

@@ -108,7 +108,9 @@ func TestProxy(t *testing.T) {
 		syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
 	}()
 
-	start(&mockConfig, &tenant, &wit, idler, store)
+	clusters := make(map[string]string)
+	clusters["https://api.free-stg.openshift.com/"] = "1b7d.free-stg.openshiftapps.com"
+	start(&mockConfig, &tenant, &wit, idler, store, clusters)
 
 	// TODO - Test an actual workflow by triggering some of the MockURLs
 

internal/clients/idler.go

@@ -7,9 +7,10 @@ import (
 	"net/http"
 	"strings"
 
+	"github.com/fabric8-services/fabric8-jenkins-proxy/internal/util"
+	"github.com/fabric8-services/fabric8-jenkins-proxy/internal/util/logging"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
-	"github.com/fabric8-services/fabric8-jenkins-proxy/internal/util"
 )
 
 const (
@@ -24,12 +25,20 @@ type status struct {
 	IsIdle bool `json:"is_idle"`
 }
 
+// clusterView is a view of the cluster topology which only includes the OpenShift API URL and the application DNS for this
+// cluster.
+type clusterView struct {
+	APIURL string
+	AppDNS string
+}
+
 type IdlerService interface {
 	IsIdle(tenant string, openShiftAPIURL string) (bool, error)
 	UnIdle(tenant string, openShiftAPIURL string) error
+	Clusters() (map[string]string, error)
 }
 
-//Idler is a simple client for Idler
+// idler is a hand-rolled Idler client using plain HTTP requests.
 type idler struct {
 	idlerApi string
 }
@@ -40,8 +49,8 @@ func NewIdler(url string) IdlerService {
 	}
 }
 
-// IsIdle returns true if the Jenkins instance for the specified tenant is idled. False otherwise.
-func (i idler) IsIdle(tenant string, openShiftAPIURL string) (bool, error) {
+// IsIdle returns true if the Jenkins instance for the specified tenant is idled, false otherwise.
+func (i *idler) IsIdle(tenant string, openShiftAPIURL string) (bool, error) {
 	namespace := tenant
 	if !strings.HasSuffix(tenant, namespaceSuffix) {
 		namespace = tenant + namespaceSuffix
@@ -57,7 +66,7 @@ func (i idler) IsIdle(tenant string, openShiftAPIURL string) (bool, error) {
 	q.Add(OpenShiftAPIParam, util.EnsureSuffix(openShiftAPIURL, "/"))
 	req.URL.RawQuery = q.Encode()
 
-	log.WithField("request", req).Debug("Calling Idler API")
+	log.WithFields(log.Fields{"request": logging.FormatHTTPRequestWithSeparator(req, " "), "type": "isidle"}).Debug("Calling Idler API")
 
 	client := &http.Client{}
 	resp, err := client.Do(req)
@@ -83,12 +92,25 @@ func (i idler) IsIdle(tenant string, openShiftAPIURL string) (bool, error) {
 }
 
 // Initiates un-idling of the Jenkins instance for the specified tenant.
-func (i idler) UnIdle(tenant string, openShiftAPIURL string) error {
+func (i *idler) UnIdle(tenant string, openShiftAPIURL string) error {
 	namespace := tenant
 	if !strings.HasSuffix(tenant, namespaceSuffix) {
 		namespace = tenant + namespaceSuffix
 	}
-	resp, err := http.Get(fmt.Sprintf("%s/api/idler/unidle/%s", i.idlerApi, namespace))
+
+	req, err := http.NewRequest("GET", fmt.Sprintf("%s/api/idler/unidle/%s", i.idlerApi, namespace), nil)
+	if err != nil {
+		return err
+	}
+
+	q := req.URL.Query()
+	q.Add(OpenShiftAPIParam, util.EnsureSuffix(openShiftAPIURL, "/"))
+	req.URL.RawQuery = q.Encode()
+
+	log.WithFields(log.Fields{"request": logging.FormatHTTPRequestWithSeparator(req, " "), "type": "unidle"}).Debug("Calling Idler API")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
 	if err != nil {
 		return err
 	}
@@ -100,3 +122,40 @@ func (i idler) UnIdle(tenant string, openShiftAPIURL string) error {
 		return errors.New(fmt.Sprintf("unexpected status code '%d' as response to unidle call.", resp.StatusCode))
 	}
 }
+
+// Clusters returns a map which maps the OpenShift API URL to the application DNS for this cluster. An empty map together with
+// an error is returned if an error occurs.
+func (i *idler) Clusters() (map[string]string, error) {
+	var clusters = make(map[string]string)
+
+	req, err := http.NewRequest("GET", fmt.Sprintf("%s/api/idler/cluster", i.idlerApi), nil)
+	if err != nil {
+		return clusters, err
+	}
+
+	log.WithFields(log.Fields{"request": logging.FormatHTTPRequestWithSeparator(req, " "), "type": "cluster"}).Debug("Calling Idler API")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return clusters, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return clusters, err
+	}
+
+	var clusterViews []clusterView
+	err = json.Unmarshal(body, &clusterViews)
+	if err != nil {
+		return clusters, err
+	}
+
+	for _, clusterView := range clusterViews {
+		clusters[clusterView.APIURL] = clusterView.AppDNS
+	}
+
+	return clusters, nil
+}

internal/configuration/configuration.go

@@ -61,9 +61,6 @@ type Configuration interface {
 	// GetDebugMode returns if debug mode should be enabled as set via default, config file, or environment variable
 	GetDebugMode() bool
 
-	// GetClusters returns map of OSO clusters apiURL -> DNS suffix for route generation
-	GetClusters() map[string]string
-
 	// String returns a string representation of the configuration
 	String() string
 }

internal/configuration/env_config.go

@@ -1,7 +1,6 @@
 package configuration
 
 import (
-	"encoding/json"
 	"fmt"
 	"github.com/fabric8-services/fabric8-jenkins-proxy/internal/util"
 	"github.com/pkg/errors"
@@ -52,7 +51,6 @@ func init() {
 	settings["GetIndexPath"] = Setting{"JC_INDEX_PATH", defaultIndexPath, []func(interface{}, string) error{util.IsNotEmpty}}
 	settings["GetMaxRequestRetry"] = Setting{"JC_MAX_REQUEST_RETRY", defaultMaxRequestRetry, []func(interface{}, string) error{util.IsInt}}
 	settings["GetDebugMode"] = Setting{"JC_DEBUG_MODE", defaultDebugMode, []func(interface{}, string) error{util.IsBool}}
-	settings["GetOsoClusters"] = Setting{"JC_OSO_CLUSTERS", "", []func(interface{}, string) error{util.IsNotEmpty}}
 }
 
 type Setting struct {
@@ -78,11 +76,6 @@ func NewConfiguration() (Configuration, error) {
 	}
 
 	config := EnvConfig{}
-	err := config.loadClusters()
-	if err != nil {
-		return nil, errors.New("Unable to load OSO cluster settings.")
-	}
-
 	return &config, nil
 }
 
@@ -247,11 +240,6 @@ func (c *EnvConfig) GetDebugMode() bool {
 	return b
 }
 
-// GetClusters returns map of OSO clusters apiURL -> DNS suffix for route generation
-func (c *EnvConfig) GetClusters() map[string]string {
-	return c.clusters
-}
-
 func (c *EnvConfig) String() string {
 	config := map[string]interface{}{}
 	for key, setting := range settings {
@@ -269,12 +257,6 @@ func (c *EnvConfig) String() string {
 	return fmt.Sprintf("%v", config)
 }
 
-func (c *EnvConfig) loadClusters() error {
-	data := getConfigValueFromEnv("GetOsoClusters")
-	err := json.Unmarshal([]byte(data), &c.clusters)
-	return err
-}
-
 // Verify checks whether all needed config options are set
 func verifyEnv() util.MultiError {
 	var errors util.MultiError