diff --git a/.gitignore b/.gitignore index 7e1ec244..1d479369 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +/richCase/rich.gocd.yaml +/simpleInvalidCase/simple-invalid.gocd.yaml /simpleCase/simple.gocd.yaml *.class diff --git a/README.md b/README.md index e38717a6..c6421de6 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,10 @@ # gocd-yaml-config-plugin -Plugin to declare [Go's](go.cd) pipelines and environments configuration in YAML. - -With this plugin and GoCD `>= 16.7.0` you can keep your pipeline and environments - configuration in source control. +[GoCD](go.cd) server plugin to keep **pipelines** and **environments** +configuration in source-control in [YAML](http://www.yaml.org/). +See [this document](https://docs.google.com/document/d/1_eGZaqIz9ydnYQJ_Xrcb3obXc-T6jIfV_pgZQNCydVk/pub) +to find out what are GoCD configuration repositories. This is second GoCD configuration plugin, which enhances some of shortcomings of [JSON configuration plugin](https://github.com/tomzo/gocd-json-config-plugin) @@ -16,6 +16,27 @@ This is second GoCD configuration plugin, which enhances some of shortcomings of * **Comments in configuration files** - YAML supports comments, so you can explain why pipeline/environment it is configured like this. +## Some highlights + + * Shorter syntax for declaring [single-job stages](#single-job-stage) + * Very short syntax for [declaring tasks with script executor plugin](#script) + +## Setup + +Download plugin from releases page and place in your Go server (`>= 16.7.0`) +in `plugins/external` directory. + +Add `config-repos` element right above first ``. Then you can +add any number of YAML configuration repositories as such: + +```xml + + + + + +``` + ### Example More examples are in [test resources](src/test/resources/examples/). @@ -77,31 +98,16 @@ pipelines: - script: ./build.sh ci ``` -## Setup - -Download plugin from releases page and place in you Go server (`>= 16.7.0`) -in `plugins/external` directory. - -Add `config-repos` element right above first ``. Then you can -add any number of YAML configuration repositories as such: - -```xml - - - - - -``` - # Specification See [official GoCD XML configuration reference](https://docs.go.cd/current/configuration/configuration_reference.html) -for details about each element. +for details about each element. Below is a reference of format supported by this plugin. +Feel free to improve it! 1. [Environment](#environment) 1. [Environment variables](#environment-variables) 1. [Pipeline](#pipeline) - * [Mingle](#mingle) + * [Tabs](#tabs) * [Tracking tool](#tracking-tool) * [Timer](#timer) 1. [Stage](#stage) @@ -154,9 +160,9 @@ All elements available on a pipeline object are: * `group` * `label_template` * `locking` - * `tracking_tool` or `mingle` - * `timer` - * `environment_variables` + * [tracking_tool](#tracking-tool) or `mingle` + * [timer](#timer) + * [environment_variables](#environment-variables) * `secure_variables` * [materials](#materials) * [stages](#stage) @@ -186,12 +192,28 @@ Please note: * [parameters](https://docs.go.cd/current/configuration/configuration_reference.html#params) are not supported * pipeline declares a group to which it belongs +### Tracking tool + +```yaml +tracking_tool: + link: "http://your-trackingtool/yourproject/${ID}" + regex: "evo-(\\d+)" +``` + +### Timer + +```yaml +timer: + spec: "0 15 10 * * ? *" + only_on_changes: yes +``` + ## Stage A minimal stage must contain `jobs:` element or `tasks:` in [single-job stage case](single-job-stage). ```yaml build: - jobbs: + jobs: firstJob: ... secondJob: @@ -240,6 +262,65 @@ approval: ## Job +[Job](https://docs.go.cd/current/configuration/configuration_reference.html#job) is a hash starting with jobs name: + +```yaml +test: + run_instances: 7 + environment_variables: + LD_LIBRARY_PATH: . + tabs: + test: results.xml + resources: + - linux + artifacts: + - test: + source: src + destination: dest + - build: + source: bin + properties: + perf: + source: test.xml + xpath: "substring-before(//report/data/all/coverage[starts-with(@type,\u0027class\u0027)]/@value, \u0027%\u0027)" + tasks: + ... +``` + +### Run many instances + +Part of job object can be [number of job to runs](https://docs.go.cd/current/advanced_usage/admin_spawn_multiple_jobs.html): +```yaml +run_instances: 7 +``` +Or to run on all agents: +```yaml +run_instances: all +``` + +### Tabs + +Tabs are a hash with `: ` pairs. +Path should exist in Go servers artifacts. +```yaml +tabs: + tests: test-reports/index.html + gauge: functional-reports/index.html +``` + +### Property + +Job can have properties, declared as a hash: +```yaml +properties: + cov1: # this is the name of property + source: test.xml + xpath: "substring-before(//report/data/all/coverage[starts-with(@type,\u0027class\u0027)]/@value, \u0027%\u0027)" + performance.ind1.mbps: + source: PerfTestReport.xml + xpath: "//PerformanceSuiteReport/WriteOnly/MBps" +``` + ### Single job stage A common use case is that stage has only one job. This plugin provides a shorthand @@ -502,6 +583,23 @@ Above executes a **single line** script: ./build.sh compile && make test ``` +## Environment + +*NOTE: The agents should be a guid, which is currently impossible to get for user* + +```yaml +testing: + environment_variables: + DEPLOYMENT: testing + secure_variables: + ENV_PASSWORD: "s&Du#@$xsSa" + pipelines: + - example-deploy-testing + - build-testing + agents: + - 123 +``` + ### Environment variables [Environment variables](https://docs.go.cd/current/configuration/configuration_reference.html#environmentvariables) diff --git a/src/main/java/cd/go/plugin/config/yaml/RootParser.java b/src/main/java/cd/go/plugin/config/yaml/RootParser.java deleted file mode 100644 index b293d36d..00000000 --- a/src/main/java/cd/go/plugin/config/yaml/RootParser.java +++ /dev/null @@ -1,16 +0,0 @@ -package cd.go.plugin.config.yaml; - -import com.esotericsoftware.yamlbeans.YamlReader; - -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.Map; - -public class RootParser { - public JsonConfigCollection parseString(InputStreamReader yaml) throws IOException { - YamlReader reader = new YamlReader(yaml); - Object object = reader.read(); - return new JsonConfigCollection(); - } - -} diff --git a/src/test/java/cd/go/plugin/config/yaml/JsonObjectMatcher.java b/src/test/java/cd/go/plugin/config/yaml/JsonObjectMatcher.java index 8d42b1e0..da04781f 100644 --- a/src/test/java/cd/go/plugin/config/yaml/JsonObjectMatcher.java +++ b/src/test/java/cd/go/plugin/config/yaml/JsonObjectMatcher.java @@ -1,5 +1,6 @@ package cd.go.plugin.config.yaml; +import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import org.hamcrest.Description; @@ -27,12 +28,39 @@ protected boolean matchesSafely(JsonObject actual) { if(!inner.matchesSafely(otherObj)) return false; } + else if(field.getValue().isJsonArray()){ + JsonArray thisArray = field.getValue().getAsJsonArray(); + JsonArray otherArray = actual.get(field.getKey()).getAsJsonArray(); + if(otherArray == null) + return false; + if(thisArray.size() != otherArray.size()) + return false; + for(int i = 0; i < thisArray.size(); i++){ + JsonElement thisItem = thisArray.get(i); + if(!equalToAnyOther(thisItem,otherArray)) + return false; + } + } else if(!field.getValue().equals(actual.get(field.getKey()))) return false; } return true; } + private boolean equalToAnyOther(JsonElement thisItem, JsonArray otherArray) { + for(int i = 0; i < otherArray.size();i++) { + JsonElement otherItem = otherArray.get(i); + if (thisItem.isJsonObject()) { + if (!otherItem.isJsonObject()) + continue; + if (new JsonObjectMatcher(thisItem.getAsJsonObject()).matchesSafely(otherItem.getAsJsonObject())) + return true; + } else if (thisItem.equals(otherItem)) + return true; + } + return false; + } + @Override public void describeTo(Description description) { description.appendText(expected.toString()); diff --git a/src/test/java/cd/go/plugin/config/yaml/RootParserTest.java b/src/test/java/cd/go/plugin/config/yaml/RootParserTest.java deleted file mode 100644 index 813c1485..00000000 --- a/src/test/java/cd/go/plugin/config/yaml/RootParserTest.java +++ /dev/null @@ -1,21 +0,0 @@ -package cd.go.plugin.config.yaml; - -import com.esotericsoftware.yamlbeans.YamlException; -import com.esotericsoftware.yamlbeans.YamlReader; -import org.apache.commons.io.IOUtils; -import org.junit.Test; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.Map; - -import static org.junit.Assert.*; - -public class RootParserTest { - @Test - public void shouldReadSimpleFile() throws IOException { - YamlReader reader = new YamlReader(TestUtils.createReader("examples/simple.gocd.yaml")); - Object object = reader.read(); - } -} \ No newline at end of file diff --git a/src/test/java/cd/go/plugin/config/yaml/YamlConfigPluginIntegrationTest.java b/src/test/java/cd/go/plugin/config/yaml/YamlConfigPluginIntegrationTest.java index e3b7ffcb..447d6561 100644 --- a/src/test/java/cd/go/plugin/config/yaml/YamlConfigPluginIntegrationTest.java +++ b/src/test/java/cd/go/plugin/config/yaml/YamlConfigPluginIntegrationTest.java @@ -20,6 +20,7 @@ import static cd.go.plugin.config.yaml.TestUtils.getResourceAsStream; import static cd.go.plugin.config.yaml.TestUtils.readJsonObject; import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsCollectionContaining.hasItem; import static org.junit.Assert.*; import static org.mockito.Matchers.any; import static org.mockito.Mockito.*; @@ -113,7 +114,7 @@ public void shouldRespondSuccessToParseDirectoryRequestWhenEmpty() throws Unhand @Test public void shouldRespondSuccessToParseDirectoryRequestWhenSimpleCaseFile() throws UnhandledRequestTypeException, IOException { - setupSimpleCase(); + setupCase("simpleCase", "simple"); DefaultGoPluginApiRequest parseDirectoryRequest = new DefaultGoPluginApiRequest("configrepo","1.0","parse-directory"); String requestBody = "{\n" + @@ -132,12 +133,54 @@ public void shouldRespondSuccessToParseDirectoryRequestWhenSimpleCaseFile() thro assertThat(responseJsonObject,is(new JsonObjectMatcher(expected))); } - private void setupSimpleCase() throws IOException { - File simpleCase = new File("simpleCase"); - FileUtils.deleteDirectory(simpleCase); - FileUtils.forceMkdir(simpleCase); - File simpleFile = new File("simpleCase/simple.gocd.yaml"); - InputStream in = getResourceAsStream("examples/simple.gocd.yaml"); + @Test + public void shouldRespondSuccessToParseDirectoryRequestWhenRichCaseFile() throws UnhandledRequestTypeException, IOException { + setupCase("richCase", "rich"); + + DefaultGoPluginApiRequest parseDirectoryRequest = new DefaultGoPluginApiRequest("configrepo","1.0","parse-directory"); + String requestBody = "{\n" + + " \"directory\":\"richCase\",\n" + + " \"configurations\":[]\n" + + "}"; + parseDirectoryRequest.setRequestBody(requestBody); + + GoPluginApiResponse response = plugin.handle(parseDirectoryRequest); + assertThat(response.responseCode(), is(DefaultGoPluginApiResponse.SUCCESS_RESPONSE_CODE)); + JsonObject responseJsonObject = getJsonObjectFromResponse(response); + assertThat(responseJsonObject.get("errors"), Is.is(new JsonArray())); + JsonArray pipelines = responseJsonObject.get("pipelines").getAsJsonArray(); + assertThat(pipelines.size(),is(1)); + JsonObject expected = (JsonObject)readJsonObject("examples.out/rich.gocd.json"); + assertThat(responseJsonObject,is(new JsonObjectMatcher(expected))); + } + + @Test + public void shouldRespondSuccessWithErrorMessagesToParseDirectoryRequestWhenSimpleInvalidCaseFile() throws UnhandledRequestTypeException, IOException { + setupCase("simpleInvalidCase", "simple-invalid"); + + DefaultGoPluginApiRequest parseDirectoryRequest = new DefaultGoPluginApiRequest("configrepo","1.0","parse-directory"); + String requestBody = "{\n" + + " \"directory\":\"simpleInvalidCase\",\n" + + " \"configurations\":[]\n" + + "}"; + parseDirectoryRequest.setRequestBody(requestBody); + + GoPluginApiResponse response = plugin.handle(parseDirectoryRequest); + assertThat(response.responseCode(), is(DefaultGoPluginApiResponse.SUCCESS_RESPONSE_CODE)); + JsonObject responseJsonObject = getJsonObjectFromResponse(response); + JsonArray errors = (JsonArray) responseJsonObject.get("errors"); + JsonArray pipelines = responseJsonObject.get("pipelines").getAsJsonArray(); + assertThat(pipelines.size(),is(0)); + assertThat(errors.get(0).getAsJsonObject().getAsJsonPrimitive("message").getAsString(),is("Failed to parse pipeline pipe1; expected a hash of pipeline materials")); + assertThat(errors.get(0).getAsJsonObject().getAsJsonPrimitive("location").getAsString(),is("simple-invalid.gocd.yaml")); + } + + private void setupCase(String folder, String caseName) throws IOException { + File caseFolder = new File(folder); + FileUtils.deleteDirectory(caseFolder); + FileUtils.forceMkdir(caseFolder); + File simpleFile = new File(folder, caseName + ".gocd.yaml"); + InputStream in = getResourceAsStream("examples/" + caseName + ".gocd.yaml"); OutputStream out = new FileOutputStream(simpleFile); IOUtils.copy(in,out); in.close(); diff --git a/src/test/resources/examples.out/rich.gocd.json b/src/test/resources/examples.out/rich.gocd.json new file mode 100644 index 00000000..e83e78b6 --- /dev/null +++ b/src/test/resources/examples.out/rich.gocd.json @@ -0,0 +1,122 @@ +{ + "target_version" : 1, + "errors" : [], + "environments" : [], + "pipelines" : [ + { + "location" : "rich.gocd.yaml", + "name" : "pipe2", + "group" : "rich", + "label_template": "${mygit[:8]}", + "enable_pipeline_locking": true, + "tracking_tool": { + "link": "http://your-trackingtool/yourproject/${ID}", + "regex": "evo-(\\d+)" + }, + "timer" : { + "spec": "0 0 22 ? * MON-FRI", + "only_on_changes" : true + }, + "materials" : [ + { + "name" : "mygit", + "type" : "git", + "url" : "http://my.example.org/mygit.git", + "branch" : "ci" + }, + { + "name" : "upstream", + "type" : "dependency", + "pipeline" : "pipe2", + "stage" : "test" + } + ], + "stages" : [ + { + "name" : "build", + "clean_working_directory" : true, + "approval" : { + "type": "manual", + "roles" : [ + "manager" + ] + }, + "jobs" : [ + { + "name" : "csharp", + "run_instance_count": 3, + "resources" : [ + "net45" + ], + "artifacts" : [ + { + "type" : "build", + "source" : "bin/", + "destination" : "build" + }, + { + "type" : "test", + "source" : "tests/", + "destination" : "test-reports/" + } + ], + "tabs" : [ + { + "name" : "report", + "path" : "test-reports/index.html" + } + ], + "environment_variables" : [ + { + "name" : "MONO_PATH", + "value" : "/usr/bin/local/mono" + }, + { + "name" : "PASSWORD", + "encrypted_value" : "s&Du#@$xsSa" + } + ], + "properties" : [ + { + "name" : "perf", + "source" : "test.xml", + "xpath" : "substring-before(//report/data/all/coverage[starts-with(@type,\u0027class\u0027)]/@value, \u0027%\u0027)" + } + ], + "tasks" : [ + { + "type" : "fetch", + "pipeline" : "pipe2", + "stage" : "build", + "job" : "test", + "source" : "test-bin/", + "destination" : "bin/" + }, + { + "type" : "exec", + "command" : "make", + "arguments" : [ + "VERBOSE=true" + ] + }, + { + "type": "plugin", + "configuration": [ + { + "key": "script", + "value": "./build.sh ci" + } + ], + "plugin_configuration": { + "id": "script-executor", + "version": "1" + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/src/test/resources/examples/rich.gocd.yaml b/src/test/resources/examples/rich.gocd.yaml index 4ed4a88c..e6841ad1 100644 --- a/src/test/resources/examples/rich.gocd.yaml +++ b/src/test/resources/examples/rich.gocd.yaml @@ -3,6 +3,7 @@ pipelines: # tells plugin that here are pipelines by name pipe2: group: rich label_template: "${mygit[:8]}" + locking: on tracking_tool: link: "http://your-trackingtool/yourproject/${ID}" regex: "evo-(\\d+)" @@ -21,14 +22,14 @@ pipelines: # tells plugin that here are pipelines by name stage: test stages: - build: # name of stage - clean: true + clean_workspace: true approval: type: manual roles: - manager jobs: csharp: # name of the job - run_instance_count: 3 + run_instances: 3 resources: - net45 artifacts: @@ -57,7 +58,7 @@ pipelines: # tells plugin that here are pipelines by name destination: bin/ - exec: # indicates type of task command: make - args: + arguments: - "VERBOSE=true" # shorthand for script-executor plugin - script: ./build.sh ci diff --git a/src/test/resources/examples/simple-invalid.gocd.yaml b/src/test/resources/examples/simple-invalid.gocd.yaml new file mode 100644 index 00000000..8c6ca3d9 --- /dev/null +++ b/src/test/resources/examples/simple-invalid.gocd.yaml @@ -0,0 +1,15 @@ +# simple.gocd.yaml +pipelines: + pipe1: + group: simple + materials: + # materials should be a hash - bogus ! + - mygit: + git: http://my.example.org/mygit.git + stages: + - build: # name of stage + jobs: + build: # name of the job + tasks: + - exec: # indicates type of task + command: make