From 2f11c0c8b25083b4ccd049103cf4c36835bafc2b Mon Sep 17 00:00:00 2001 From: Kirill Date: Fri, 19 Nov 2021 17:45:06 +0300 Subject: [PATCH] feat: implement info/refs slice Add `InfoRefs` slice implementation which returns server metadata and supported command that server can handle. Based on this response git client decide what command to execute next. Add `GitResponseOutput` to format ASCI text as git response with length prefixes and part-end symbols. Add `main` entry point to debug Artipie git server on localhost. Update integration test `GitITCase`: setup bare repo on storage, updated `ls-remote` test, added correct assertiion for `ls-remote` test. Ticket: #1 Ticket: #11 --- pom.xml | 87 ++++----------- .../com/artipie/git/GitResponseOutput.java | 67 ++++++++++++ src/main/java/com/artipie/git/GitSlice.java | 55 +++++++--- .../java/com/artipie/git/InfoRefsSlice.java | 93 ++++++++++++++++ .../com/artipie/git/ReceivePackSlice.java | 17 +-- .../java/com/artipie/git/UploadPackSlice.java | 15 --- src/main/resources/log4j.properties | 7 ++ src/test/java/com/artipie/git/Cmd.java | 101 ++++++++++++++++++ src/test/java/com/artipie/git/GitITCase.java | 38 ++++++- .../artipie/git/GitResponseOutputTest.java | 81 ++++++++++++++ 10 files changed, 450 insertions(+), 111 deletions(-) create mode 100644 src/main/java/com/artipie/git/GitResponseOutput.java create mode 100644 src/main/java/com/artipie/git/InfoRefsSlice.java create mode 100644 src/main/resources/log4j.properties create mode 100644 src/test/java/com/artipie/git/Cmd.java create mode 100644 src/test/java/com/artipie/git/GitResponseOutputTest.java diff --git a/pom.xml b/pom.xml index bcd4ca9..304219e 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ SOFTWARE. com.artipie ppom - 1.0.4 + 1.1.0 git-adapter 1.0-SNAPSHOT @@ -35,6 +35,18 @@ SOFTWARE. An Artipie adapter for git repositories https://github.com/artipie/git-adapter 2021 + + + g4s8 + Kirill Che. + g4s8.public@gmail.com + Artipie + https://www.artipie.com + + maintainer + + + MIT @@ -92,7 +104,6 @@ SOFTWARE. com.artipie vertx-server 0.5 - test org.testcontainers @@ -103,83 +114,29 @@ SOFTWARE. log4j log4j - test + org.slf4j slf4j-log4j12 - test + org.slf4j slf4j-api 1.8.0-alpha2 - test + org.junit.jupiter junit-jupiter-params test + + commons-io + commons-io + 2.11.0 + test + - - - - - maven-surefire-plugin - 3.0.0-M5 - - false - false - - - - - - - com.jcabi - jcabi-maven-plugin - - - jcabi-versionalize-packages - none - - - - - maven-failsafe-plugin - - - - integration-test - verify - - - - - - - - - qulice - - - - com.qulice - qulice-maven-plugin - 0.19.4 - - - findbugs:.* - duplicatefinder:.* - xml:/src/it/settings.xml - dependencies:.* - checkstyle:.*/src/test/resources/.* - - - - - - - diff --git a/src/main/java/com/artipie/git/GitResponseOutput.java b/src/main/java/com/artipie/git/GitResponseOutput.java new file mode 100644 index 0000000..f37856f --- /dev/null +++ b/src/main/java/com/artipie/git/GitResponseOutput.java @@ -0,0 +1,67 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2021 artipie.com + * https://github.com/artipie/git-adapter/LICENSE.txt + */ +package com.artipie.git; + +import java.io.IOException; +import java.io.Writer; + +/** + * Git response formatter writes binary data to + * wrapped {@link Writer} with correct identations and + * line prefixes. + *

+ * Git response data consists of any number of part, each part + * ends with empty line (no data binary line).
+ * Git data line contains ASCI characters.
+ * Each data line is prefixed with 4-byte length of full line in hex ASCI + * format, e.g. text hello will be printed as + * {@code `000ahello\n`: `000a = hex(4 + len("hello") + 1)}
+ * Each part ends with empty data line: {@code 0000}. + * @since 1.0 + */ +final class GitResponseOutput { + + /** + * End part symbols. + */ + private static final char[] END_PART = new char[]{'0', '0', '0', '0'}; + + /** + * Output writer. + */ + private final Writer out; + + /** + * New git output. + * @param writer Writer + */ + GitResponseOutput(final Writer writer) { + this.out = writer; + } + + /** + * Push new line to output, format it as git data line. + * @param line ASCI text line + * @throws IOException On IO error + * @checkstyle MagicNumberCheck (10 lines) + */ + void pushLine(final String line) throws IOException { + final char[] src = line.toCharArray(); + final char[] res = new char[src.length + 5]; + final char[] len = String.format("%04x", res.length).toCharArray(); + System.arraycopy(len, 0, res, 0, len.length); + System.arraycopy(src, 0, res, len.length, src.length); + res[res.length - 1] = '\n'; + this.out.write(res); + } + + /** + * Write end part symbol. + * @throws IOException Of IO error + */ + void endPart() throws IOException { + this.out.write(GitResponseOutput.END_PART); + } +} diff --git a/src/main/java/com/artipie/git/GitSlice.java b/src/main/java/com/artipie/git/GitSlice.java index df4fdfc..2c0d003 100644 --- a/src/main/java/com/artipie/git/GitSlice.java +++ b/src/main/java/com/artipie/git/GitSlice.java @@ -5,6 +5,7 @@ package com.artipie.git; import com.artipie.asto.Storage; +import com.artipie.asto.fs.FileStorage; import com.artipie.http.Slice; import com.artipie.http.rq.RequestLineFrom; import com.artipie.http.rq.RqParams; @@ -12,7 +13,14 @@ import com.artipie.http.rt.RtRule; import com.artipie.http.rt.RtRulePath; import com.artipie.http.rt.SliceRoute; +import com.artipie.http.slice.LoggingSlice; +import com.artipie.vertx.VertxSliceServer; +import com.jcabi.log.Logger; +import io.vertx.reactivex.core.Vertx; +import java.nio.file.Path; +import java.util.Arrays; import java.util.Map.Entry; +import java.util.TreeSet; /** * Git main entry point. @@ -21,6 +29,8 @@ *

* * @since 1.0 + * @checkstyle MethodBodyCommentsCheck (500 lines) + * @checkstyle ClassDataAbstractionCouplingCheck (500 lines) */ public final class GitSlice extends Slice.Wrap { @@ -32,13 +42,6 @@ public final class GitSlice extends Slice.Wrap { public GitSlice(final Storage storage) { super( new SliceRoute( - new RtRulePath( - new RtRule.All( - ReceivePackSlice.RT_RULE, - ByMethodsRule.Standard.GET - ), - new ReceivePackSlice.InfoRefSlice() - ), new RtRulePath( new RtRule.All( ReceivePackSlice.RT_RULE, @@ -49,21 +52,49 @@ public GitSlice(final Storage storage) { new RtRulePath( new RtRule.All( UploadPackSlice.RT_RULE, - ByMethodsRule.Standard.GET + ByMethodsRule.Standard.POST ), - new UploadPackSlice.InfoRefSlice() + new UploadPackSlice() ), new RtRulePath( new RtRule.All( - UploadPackSlice.RT_RULE, - ByMethodsRule.Standard.POST + new RtRule.ByPath("/info/refs"), + ByMethodsRule.Standard.GET ), - new UploadPackSlice() + new InfoRefsSlice( + "git/artipie", + new TreeSet<>( + Arrays.asList( + "ls-refs=unborn", + "fetch=shallow wait-for-done filter" + ) + ) + ) ) ) ); } + /** + * Main entry point for debugging with git. + * @param args First arg is a path to git dir + */ + public static void main(final String... args) { + final String repo; + if (args.length > 0) { + repo = args[0]; + } else { + repo = "/tmp/artipie-git"; + } + final VertxSliceServer server = new VertxSliceServer( + Vertx.vertx(), + new LoggingSlice(new GitSlice(new FileStorage(Path.of(repo)))), + 8080 + ); + final int port = server.start(); + Logger.info(GitSlice.class, "Artipie git server started at http://localhost:%d", port); + } + /** * Routing rule by service name. * diff --git a/src/main/java/com/artipie/git/InfoRefsSlice.java b/src/main/java/com/artipie/git/InfoRefsSlice.java new file mode 100644 index 0000000..664353d --- /dev/null +++ b/src/main/java/com/artipie/git/InfoRefsSlice.java @@ -0,0 +1,93 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2021 artipie.com + * https://github.com/artipie/git-adapter/LICENSE.txt + */ +package com.artipie.git; + +import com.artipie.asto.ArtipieIOException; +import com.artipie.asto.Content; +import com.artipie.http.ArtipieHttpException; +import com.artipie.http.Response; +import com.artipie.http.Slice; +import com.artipie.http.headers.ContentType; +import com.artipie.http.rq.RequestLineFrom; +import com.artipie.http.rq.RqParams; +import com.artipie.http.rs.RsStatus; +import com.artipie.http.rs.RsWithBody; +import com.artipie.http.rs.RsWithHeaders; +import com.artipie.http.rs.StandardRs; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.nio.ByteBuffer; +import java.util.Map.Entry; +import java.util.Set; +import org.reactivestreams.Publisher; + +/** + * Slice to handle {@code /info/refs} - it's used + * to send metadata about git server, it shows supported commands + * services, versions, etc. + * @since 1.0 + * @checkstyle ClassDataAbstractionCouplingCheck (500 lines) + * @checkstyle MethodBodyCommentsCheck (500 lines) + */ +final class InfoRefsSlice implements Slice { + + /** + * Server agent name. + */ + private final String agent; + + /** + * Supported commands. + */ + private final Set commands; + + /** + * New slice. + * @param agent Server agent name + * @param commands Supported commands with annotations + */ + InfoRefsSlice(final String agent, final Set commands) { + this.agent = agent; + this.commands = commands; + } + + @Override + public Response response(final String line, final Iterable> headers, + final Publisher body) { + final String service = new RqParams(new RequestLineFrom(line).uri()).value("service") + .orElseThrow( + () -> new ArtipieHttpException( + RsStatus.BAD_REQUEST, + "service query param required" + ) + ); + // this response is very small: <1K - it doesn't consume a lot of memory + // and it could be constructed in byte-array right here + final ByteArrayOutputStream baos = new ByteArrayOutputStream(256); + try (OutputStreamWriter osw = new OutputStreamWriter(baos)) { + final GitResponseOutput gwr = new GitResponseOutput(osw); + gwr.pushLine(String.format("# service=%s", service)); + gwr.endPart(); + gwr.pushLine("version 2"); + gwr.pushLine(String.format("agent=%s", this.agent)); + for (final String cmd : this.commands) { + gwr.pushLine(cmd); + } + gwr.pushLine("server-option"); + gwr.pushLine("object-format=sha1"); + gwr.endPart(); + } catch (final IOException iex) { + throw new ArtipieHttpException(RsStatus.INTERNAL_ERROR, new ArtipieIOException(iex)); + } + return new RsWithBody( + new RsWithHeaders( + StandardRs.OK, + new ContentType("application/x-git-upload-pack-advertisement") + ), + new Content.From(baos.toByteArray()) + ); + } +} diff --git a/src/main/java/com/artipie/git/ReceivePackSlice.java b/src/main/java/com/artipie/git/ReceivePackSlice.java index 19a1c1c..004888d 100644 --- a/src/main/java/com/artipie/git/ReceivePackSlice.java +++ b/src/main/java/com/artipie/git/ReceivePackSlice.java @@ -31,7 +31,7 @@ final class ReceivePackSlice extends Slice.Wrap { /** * Service routing rule. */ - static final RtRule RT_RULE = new GitSlice.ByService("git-upload-pack"); + static final RtRule RT_RULE = new GitSlice.ByService("git-receive-pack"); /** * New Slice. @@ -39,19 +39,4 @@ final class ReceivePackSlice extends Slice.Wrap { ReceivePackSlice() { super(new SliceSimple(new RsWithStatus(RsStatus.NOT_IMPLEMENTED))); } - - /** - * A slice to return info references as a first phase of {@code receive-pack}. - * - * @since 1.0 - */ - static final class InfoRefSlice extends Slice.Wrap { - - /** - * New info refs slice. - */ - InfoRefSlice() { - super(new SliceSimple(new RsWithStatus(RsStatus.NOT_IMPLEMENTED))); - } - } } diff --git a/src/main/java/com/artipie/git/UploadPackSlice.java b/src/main/java/com/artipie/git/UploadPackSlice.java index 7f17fb5..af54d4f 100644 --- a/src/main/java/com/artipie/git/UploadPackSlice.java +++ b/src/main/java/com/artipie/git/UploadPackSlice.java @@ -40,19 +40,4 @@ final class UploadPackSlice extends Slice.Wrap { UploadPackSlice() { super(new SliceSimple(new RsWithStatus(RsStatus.NOT_IMPLEMENTED))); } - - /** - * References info phase for upload pack. - * - * @since 1.0 - */ - static final class InfoRefSlice extends Slice.Wrap { - - /** - * Ne info refs slice. - */ - InfoRefSlice() { - super(new SliceSimple(new RsWithStatus(RsStatus.NOT_IMPLEMENTED))); - } - } } diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties new file mode 100644 index 0000000..ba75441 --- /dev/null +++ b/src/main/resources/log4j.properties @@ -0,0 +1,7 @@ +log4j.rootLogger=WARN, CONSOLE + +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.layout=com.jcabi.log.MulticolorLayout +log4j.appender.CONSOLE.layout.ConversionPattern=[%color{%p}] %t %c: %m%n + +log4j.logger.com.artipie.git=DEBUG diff --git a/src/test/java/com/artipie/git/Cmd.java b/src/test/java/com/artipie/git/Cmd.java new file mode 100644 index 0000000..789c45a --- /dev/null +++ b/src/test/java/com/artipie/git/Cmd.java @@ -0,0 +1,101 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2021 artipie.com + * https://github.com/artipie/git-adapter/LICENSE.txt + */ +package com.artipie.git; + +import com.artipie.asto.misc.UncheckedConsumer; +import com.jcabi.log.Logger; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.function.Consumer; +import org.apache.commons.io.IOUtils; + +/** + * Local command to execute. + * @since 1.0 + */ +final class Cmd { + + /** + * Command list. + */ + private final List commands; + + /** + * New command. + * @param cmd Command list + */ + Cmd(final String... cmd) { + this.commands = Arrays.asList(cmd); + } + + /** + * Exec command with stdin string. + * @param stdin Stdin + * @return Stdout + * @throws IOException On error + */ + public String exec(final String stdin) throws IOException { + return this.patchExec( + new UncheckedConsumer<>( + proc -> { + proc.getOutputStream().write(stdin.getBytes(StandardCharsets.US_ASCII)); + proc.getOutputStream().close(); + } + ) + ); + } + + /** + * Exec command. + * @return Stdout + * @throws IOException On error + */ + public String exec() throws IOException { + return this.patchExec( + ignore -> { + } + ); + } + + /** + * Patch process and execute it. + * @param patcher Patcher func + * @return Stdout ASCI string + * @throws IOException On error + * @checkstyle ReturnCountCheck (30 lines) + */ + @SuppressWarnings("PMD.OnlyOneReturn") + private String patchExec(final Consumer patcher) throws IOException { + Logger.info(this, "$ %s", String.join(" ", this.commands)); + final Process proc = Runtime.getRuntime().exec(this.commands.toArray(new String[0])); + patcher.accept(proc); + final int exit; + try { + exit = proc.waitFor(); + } catch (final InterruptedException ignore) { + Thread.currentThread().interrupt(); + return null; + } + if (exit != 0) { + throw new IOException( + String.format( + "cmd '%s' exit with %d\n\t%s\n", + String.join(" ", this.commands), + exit, + new String( + IOUtils.toByteArray(proc.getErrorStream()), StandardCharsets.UTF_8 + ) + ) + ); + } + final String res = new String( + IOUtils.toByteArray(proc.getInputStream()), StandardCharsets.UTF_8 + ); + Logger.info(this, "> %s\n", res); + return res.trim(); + } +} diff --git a/src/test/java/com/artipie/git/GitITCase.java b/src/test/java/com/artipie/git/GitITCase.java index f08343d..9242f3d 100644 --- a/src/test/java/com/artipie/git/GitITCase.java +++ b/src/test/java/com/artipie/git/GitITCase.java @@ -7,11 +7,14 @@ import com.artipie.asto.fs.FileStorage; import com.artipie.http.slice.LoggingSlice; import com.artipie.vertx.VertxSliceServer; +import com.jcabi.log.Logger; import io.vertx.reactivex.core.Vertx; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.Path; import java.util.logging.Level; +import org.hamcrest.MatcherAssert; +import org.hamcrest.core.StringContains; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; @@ -25,6 +28,9 @@ /** * Integration test for Artipie Git server. * @since 1.0 + * @checkstyle ClassDataAbstractionCouplingCheck (500 lines) + * @checkstyle ExecutableStatementCountCheck (500 lines) + * @checkstyle MethodBodyCommentsCheck (500 lines) */ @Disabled @SuppressWarnings({"PMD.SystemPrintln", "PMD.TooManyMethods"}) @@ -49,12 +55,31 @@ final class GitITCase { @SuppressWarnings("PMD.AvoidDuplicateLiterals") void beforeEach(@TempDir final Path tmp) throws Exception { this.vertx = Vertx.vertx(); + // init bare repository with test data + final String gitdir = tmp.toAbsolutePath().toString(); + new Cmd("git", "--git-dir", gitdir, "init", "--bare").exec(); + final String hash = new Cmd( + "git", "--git-dir", gitdir, "hash-object", "-w", "--stdin" + ).exec("test\n"); + new Cmd( + "git", "--git-dir", gitdir, "update-index", "--add", + "--cacheinfo", "10644", hash, "test.txt" + ).exec(); + final String tree = new Cmd("git", "--git-dir", gitdir, "write-tree").exec(); + final String commit = new Cmd( + "git", "--git-dir", gitdir, "commit-tree", "-m", "test commit", tree + ).exec(); + new Cmd( + "git", "--git-dir", gitdir, "update-ref", "refs/heads/master", commit + ).exec(); + Logger.info(this, "initialize git bare repo at '%s'\n", gitdir); this.server = new VertxSliceServer( this.vertx, new LoggingSlice(Level.WARNING, new GitSlice(new FileStorage(tmp))) ); final int port = this.server.start(); final String base = String.format("http://host.testcontainers.internal:%d", port); + Logger.info(this, "Artipie git server started on %s\n", base); this.container = new GitContainer() .withWorkingDirectory("/w") .withEnv("GIT_CURL_VERBOSE", "1") @@ -70,18 +95,23 @@ void beforeEach(@TempDir final Path tmp) throws Exception { @AfterEach void tearDown() { + System.out.println("stopping artipie server"); if (this.server != null) { this.server.close(); } + System.out.println("closing vertx"); if (this.vertx != null) { this.vertx.close(); } + System.out.println("stopping container"); if (this.container != null) { + this.container.stop(); this.container.close(); } } @Test + @Disabled void pushToRemote() throws Exception { final String path = "/tmp/data"; // @checkstyle MagicNumberCheck (5 lines) @@ -93,18 +123,20 @@ void pushToRemote() throws Exception { this.gitUpdateIndexCache(hash, "test"); final String tree = this.gitWriteTree(); final String commit = this.gitCommitTree(tree, "test-commit"); - this.gitUpdateRef("refs/head/master", commit); + this.gitUpdateRef("refs/heads/test", commit); this.bash("git push origin refs/head/master"); } @Test + @Disabled void fetchFromRemote() { this.bash("git fetch -pvt"); } @Test void lsRemote() { - this.bash("git ls-remote"); + final String result = this.bash("git ls-remote --refs --quiet --heads"); + MatcherAssert.assertThat(result, new StringContains("refs/heads/master")); } /** @@ -154,7 +186,7 @@ private void gitUpdateRef(final String ref, final String commit) { /** * Executes a command. - * @param fmt Command format + * @param fmt Command formatStderr * @param args Format args * @return Stdout * @checkstyle ReturnCountCheck (20 lines) diff --git a/src/test/java/com/artipie/git/GitResponseOutputTest.java b/src/test/java/com/artipie/git/GitResponseOutputTest.java new file mode 100644 index 0000000..b588535 --- /dev/null +++ b/src/test/java/com/artipie/git/GitResponseOutputTest.java @@ -0,0 +1,81 @@ +/* + * The MIT License (MIT) Copyright (c) 2020-2021 artipie.com + * https://github.com/artipie/git-adapter/LICENSE.txt + */ +package com.artipie.git; + +import com.artipie.asto.misc.UncheckedConsumer; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.function.Consumer; +import org.hamcrest.MatcherAssert; +import org.hamcrest.core.IsEqual; +import org.junit.jupiter.api.Test; + +/** + * Test case for {@link GitResponseOutput}. + * @since 1.0 + */ +final class GitResponseOutputTest { + @Test + void writeDataLine() { + MatcherAssert.assertThat( + withGitOutput( + new UncheckedConsumer<>( + out -> out.pushLine("hello") + ) + ), + new IsEqual<>("000ahello\n") + ); + } + + @Test + void writeParts() { + MatcherAssert.assertThat( + withGitOutput( + new UncheckedConsumer<>( + out -> out.endPart() + ) + ), + new IsEqual<>("0000") + ); + } + + @Test + void writeMultipleParts() { + MatcherAssert.assertThat( + withGitOutput( + new UncheckedConsumer<>( + out -> { + out.pushLine("line-1"); + out.endPart(); + out.pushLine("line-02"); + out.pushLine("line-003"); + out.endPart(); + } + ) + ), + new IsEqual<>("000bline-1\n0000000cline-02\n000dline-003\n0000") + ); + } + + /** + * Perform operation with git data output and return ASCI string with result. + * @param func Consumer + * @return ASCI string + */ + private static String withGitOutput(final Consumer func) { + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + final OutputStreamWriter osw = new OutputStreamWriter(baos); + func.accept(new GitResponseOutput(osw)); + try { + osw.flush(); + } catch (final IOException iex) { + throw new UncheckedIOException(iex); + } + return new String(baos.toByteArray(), StandardCharsets.US_ASCII); + } +}