Merge branch 'main' of ssh://github.com/mlhartme/pommes

.github/workflows/maven.yml

@@ -0,0 +1,25 @@
+name: Java CI with Maven
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up JDK 17
+      uses: actions/setup-java@v3
+      with:
+        java-version: '17'
+        distribution: 'temurin'
+        cache: maven
+    - name: maven-settings-xml-action
+      uses: whelk-io/maven-settings-xml-action@v4
+      with:
+        repositories: '[{ "id": "oss-sonatype-snapshot", "name": "oss-sonatype-snapshot", "url": "https://oss.sonatype.org/content/repositories/snapshots/" }]'
+    - name: Build with Maven
+      run: mvn -B -U clean package

CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ### 3.5.0 (pending)
 
+* token handling
+  * load token from environment POMMES_REPOSITORY_TOKEN_<name> variable, not file
+  * github + gitlab
+    * load github token from "git credentials"
+    * explicitly declare "§§" if repository needs a token
 * default repositories
   * the first repository defined in config is the default repository
   * setup automatically defines default repository local = $POMMES_ROOT

src/main/java/net/oneandone/pommes/checkout/Checkout.java

@@ -63,7 +63,7 @@ public void run(Console console) throws MkdirException, Failure {
         Launcher launcher;
 
         directory.getParent().mkdirsOpt();
-        launcher = scm.scm().checkout(directory, scm.url());
+        launcher = scm.scm().checkout(scm, directory);
         console.info.println(launcher.toString());
         if (console.getVerbose()) {
             launcher.exec(console.verbose);

src/main/java/net/oneandone/pommes/cli/Environment.java

@@ -47,6 +47,7 @@ public Environment(Console console, World world) throws IOException {
         this.console = console;
         this.world = world;
         this.lib = Lib.load(world);
+        console.verbose.println("default repository: " + lib.properties().defaultRepository);
         this.lazyMaven = null;
         this.lazyCurrentPom = null;
         this.lazyExcludes = null;

src/main/java/net/oneandone/pommes/cli/Index.java

@@ -23,6 +23,8 @@
 import net.oneandone.pommes.descriptor.Descriptor;
 import net.oneandone.pommes.descriptor.ErrorDescriptor;
 import net.oneandone.pommes.repository.Repository;
+import net.oneandone.pommes.scm.Git;
+import net.oneandone.pommes.scm.Scm;
 import net.oneandone.sushi.util.Separator;
 import org.apache.lucene.document.Document;
 
@@ -66,7 +68,13 @@ public void run(Scope scope) throws Exception {
             }
             repository = null;
             for (String str : Separator.SPACE.split(entry.getValue())) {
-                if (str.startsWith("-")) {
+                if (str.equals("§§")) {
+                    String host = repo(repository, str).getTokenHost();
+                    if (host != null) {
+                        Git.UP up = Scm.GIT.getCredentials(environment.console(), environment.world().getWorking(), host);
+                        repo(repository, str).setToken(up.password());
+                    }
+                } else if (str.startsWith("-")) {
                     repo(repository, str).addExclude(str.substring(1));
                 } else if (str.startsWith("%")) {
                     repo(repository, str).addOption(str.substring(1));

src/main/java/net/oneandone/pommes/cli/Lib.java

@@ -72,9 +72,9 @@ public Lib(FileNode home) throws IOException {
         this.properties = Properties.load(configFile(home));
     }
 
-    public String tokenOpt(String repositoryName) throws IOException {
-        var file = home.join(".pommes-" + repositoryName + ".token");
-        return file.exists() ? file.readString().trim() : null;
+    public String tokenOpt(String repositoryName) {
+        // TODO merge with git credential access
+        return System.getenv("POMMES_REPOSITORY_TOKEN_" + repositoryName.toUpperCase());
     }
 
     public FileNode projectDirectory(Project project) throws IOException {

src/main/java/net/oneandone/pommes/cli/Properties.java

@@ -33,14 +33,9 @@ public static List<String> defaultConfig(Map<String, String> repositories) {
         lines = new ArrayList<>();
         lines.add("# Pommes configuration file, see https://github.com/mlhartme/pommes");
         lines.add("");
-        lines.add("# repositories for indexing");
-        if (repositories.isEmpty()) {
-            lines.add("#repository.first=url1");
-            lines.add("#repository.second=url2");
-        } else {
-            for (var entry : repositories.entrySet()) {
-                lines.add("repository." + entry.getKey() + "=" + entry.getValue());
-            }
+        lines.add("# repositories");
+        for (var entry : repositories.entrySet()) {
+            lines.add("repository." + entry.getKey() + "=" + entry.getValue());
         }
         lines.add("");
         lines.add("# query macros");
@@ -65,7 +60,7 @@ public static Properties load(FileNode file) throws IOException {
         giteaKey = null;
         queries = new HashMap<>();
         formats = new HashMap<>();
-        repositories = new HashMap<>();
+        repositories = new LinkedHashMap<>();
         props = readSequencedProperties(file);
         checkouts = file.getParent().getParent();
         for (String key : props.keySet()) {
@@ -82,7 +77,7 @@ public static Properties load(FileNode file) throws IOException {
             }
         }
         if (repositories.isEmpty()) {
-            throw new IOException("missing repositories");
+            throw new IOException("missing repositories: " + file);
         }
         return new Properties(checkouts, giteaKey, queries, formats, repositories, repositories.keySet().iterator().next());
     }

src/main/java/net/oneandone/pommes/repository/ArtifactoryRepository.java

@@ -16,44 +16,43 @@
 package net.oneandone.pommes.repository;
 
 import net.oneandone.inline.ArgumentException;
-import net.oneandone.inline.Console;
 import net.oneandone.pommes.cli.Environment;
 import net.oneandone.pommes.descriptor.Descriptor;
 import net.oneandone.sushi.fs.Node;
+import net.oneandone.sushi.fs.NodeInstantiationException;
 import net.oneandone.sushi.fs.World;
 import net.oneandone.sushi.util.Strings;
 
 import javax.json.Json;
 import javax.json.stream.JsonParser;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.PrintWriter;
 import java.net.URISyntaxException;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
 import java.util.Date;
-import java.util.concurrent.BlockingQueue;
+import java.util.List;
 
-public class ArtifactoryRepository extends Repository {
-    private static final String PROTOCOL = "artifactory:";
-
-    public static ArtifactoryRepository createOpt(Environment environment, String name, String url) {
-        if (url.startsWith(PROTOCOL)) {
-            return new ArtifactoryRepository(environment, name, url.substring(PROTOCOL.length()));
-        } else {
-            return null;
-        }
+public class ArtifactoryRepository extends Repository<Descriptor> {
+    public static ArtifactoryRepository create(Environment environment, String name, String url, PrintWriter log)
+            throws NodeInstantiationException, URISyntaxException {
+        return new ArtifactoryRepository(environment, name, url);
     }
 
     private final String url;
+    private final Node<?> root;
     private final Environment environment;
     private final World world;
     private String contextPath;
 
-    public ArtifactoryRepository(Environment environment, String name, String url) {
+    public ArtifactoryRepository(Environment environment, String name, String url) throws NodeInstantiationException, URISyntaxException {
         super(name);
         this.environment = environment;
         this.world = environment.world();
         this.url = url;
+        this.root = world.node(url);
         this.contextPath = "/artifactory/";
         if (!url.contains(contextPath)) {
             this.contextPath = "/";
@@ -73,19 +72,23 @@ public void addOption(String option) {
     }
 
     @Override
-    public void scan(BlockingQueue<Descriptor> dest, Console console) throws IOException, URISyntaxException {
-        Node listing;
-        Node root;
-
-        root = world.node(url);
-        listing = world.node(artifactory() + "api/storage/" + repositoryAndPath() + "?list&deep=1&mdTimestamps=0");
+    public List<Descriptor> list() throws IOException {
+        List<Descriptor> result = new ArrayList<>();
+        Node<?> listing = world.validNode(artifactory() + "api/storage/" + repositoryAndPath() + "?list&deep=1&mdTimestamps=0");
         try {
-            Parser.run(environment, name, listing, root, dest);
+            Parser.run(environment, name, listing, root, result);
         } catch (IOException | RuntimeException e) {
             throw e;
         } catch (Exception e) {
             throw new IOException("scanning failed: " + e.getMessage(), e);
         }
+        return result;
+    }
+
+    @Override
+    public Descriptor load(Descriptor descriptor) {
+        // TODO: currently all processing done by parser
+        return descriptor;
     }
 
     /** @return with tailing slash */
@@ -108,14 +111,13 @@ private String repositoryAndPath() {
     public static class Parser implements AutoCloseable {
         private static final SimpleDateFormat FMT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX");
 
-        public static void run(Environment environment, String repository, Node listing, Node root, BlockingQueue<Descriptor> dest) throws Exception {
+        public static void run(Environment environment, String repository, Node listing, Node root, List<Descriptor> dest) throws Exception {
             String uri;
             long size;
             Date lastModified;
             String sha1;
             int count;
             Node node;
-            Descriptor descriptor;
 
             count = 0;
             try (InputStream is = listing.newInputStream(); Parser parser = new Parser(Json.createParser(is))) {
@@ -137,7 +139,7 @@ public static void run(Environment environment, String repository, Node listing,
                     node = root.join(Strings.removeLeft(uri, "/"));
                     Descriptor.Creator m = Descriptor.match(((Node<?>) node).getName());
                     if (m != null) {
-                        dest.put(m.create(environment, node, repository, "artifactory:" + node.getPath(), sha1, null));
+                        dest.add(m.create(environment, node, repository, "artifactory:" + node.getPath(), sha1, null));
                     }
                     if (parser.eatTimestampsOpt() != JsonParser.Event.END_OBJECT) {
                         throw new IllegalStateException();

src/main/java/net/oneandone/pommes/repository/BitbucketRepository.java

@@ -18,7 +18,6 @@
 import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParser;
-import net.oneandone.inline.Console;
 import net.oneandone.pommes.cli.Environment;
 import net.oneandone.pommes.descriptor.Descriptor;
 import net.oneandone.pommes.scm.GitUrl;
@@ -36,10 +35,9 @@
 import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.BlockingQueue;
 
 /** https://developer.atlassian.com/static/rest/bitbucket-server/4.6.2/bitbucket-rest.html */
-public class BitbucketRepository extends Repository {
+public class BitbucketRepository extends Repository<Descriptor> {
     public static void main(String[] args) throws IOException {
         World world;
         Bitbucket bb;
@@ -49,14 +47,8 @@ public static void main(String[] args) throws IOException {
         System.out.println("rev: " + new String(bb.readBytes("CISOOPS", "puc", "pom.xml")));
     }
 
-    private static final String PROTOCOL = "bitbucket:";
-
-    public static BitbucketRepository createOpt(Environment environment, String name, String url) throws URISyntaxException, NodeInstantiationException {
-        if (url.startsWith(PROTOCOL)) {
-            return new BitbucketRepository(environment, name, (HttpNode) environment.world().node(url.substring(PROTOCOL.length())));
-        } else {
-            return null;
-        }
+    public static BitbucketRepository create(Environment environment, String name, String url) throws URISyntaxException, NodeInstantiationException {
+        return new BitbucketRepository(environment, name, (HttpNode) environment.world().node(url));
     }
 
     private final Environment environment;
@@ -69,10 +61,10 @@ public BitbucketRepository(Environment environment, String name, HttpNode bitbuc
     }
 
     @Override
-    public void scan(BlockingQueue<Descriptor> dest, Console console) throws IOException, InterruptedException {
+    public List<Descriptor> list() throws IOException {
+        List<Descriptor> result = new ArrayList<>();
         Bitbucket bb;
         String bbProject;
-        Descriptor descriptor;
         Descriptor.Creator m;
         byte[] bytes;
         List<String> lst;
@@ -89,11 +81,18 @@ public void scan(BlockingQueue<Descriptor> dest, Console console) throws IOExcep
                     bytes = bb.readBytes(bbProject, repo, path);
                     tmp = environment.world().memoryNode(bytes);
                     hostname = bitbucket.getRoot().getHostname();
-                    dest.put(m.create(environment, tmp, this.name, bbProject.toLowerCase() + "/" + repo + "/" + path, tmp.sha(),
+                    result.add(m.create(environment, tmp, this.name, bbProject.toLowerCase() + "/" + repo + "/" + path, tmp.sha(),
                             GitUrl.create("ssh://git@" + hostname + "/" + bbProject.toLowerCase() + "/" + repo + ".git")));
                 }
             }
         }
+        return result;
+    }
+
+    @Override
+    public Descriptor load(Descriptor descriptor) throws IOException {
+        // TODO
+        return descriptor;
     }
 
     private static class Bitbucket {

src/main/java/net/oneandone/pommes/repository/Constructor.java

@@ -0,0 +1,12 @@
+package net.oneandone.pommes.repository;
+
+import net.oneandone.pommes.cli.Environment;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.net.URISyntaxException;
+
+@FunctionalInterface
+public interface Constructor {
+    Repository create(Environment environment, String name, String url, PrintWriter log) throws URISyntaxException, IOException;
+}

src/main/java/net/oneandone/pommes/repository/GiteaProject.java

@@ -0,0 +1,40 @@
+package net.oneandone.pommes.repository;
+
+import io.gitea.ApiException;
+import io.gitea.api.RepositoryApi;
+import io.gitea.model.ContentsResponse;
+import net.oneandone.pommes.cli.Environment;
+import net.oneandone.pommes.descriptor.Descriptor;
+import net.oneandone.pommes.scm.GitUrl;
+import net.oneandone.sushi.fs.memory.MemoryNode;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.List;
+
+public record GiteaProject(String org, String repo, String ref) {
+    public Descriptor scanOpt(Environment environment, String hostname, RepositoryApi repositoryApi) throws IOException, ApiException {
+        List<ContentsResponse> lst;
+
+        lst = repositoryApi.repoGetContentsList(org, repo, ref);
+        for (ContentsResponse contents : lst) {
+            if ("file".equals(contents.getType())) {
+                Descriptor.Creator m = Descriptor.match(contents.getName());
+                if (m != null) {
+                    contents = repositoryApi.repoGetContents(org, repo, contents.getPath(), ref);
+                    String str = contents.getContent();
+                    if ("base64".equals(contents.getEncoding())) {
+                        str = new String(Base64.getDecoder().decode(str.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8);
+                    } else if (contents.getEncoding() != null) {
+                        throw new IllegalStateException(contents.getEncoding());
+                    }
+                    MemoryNode tmp = environment.world().memoryNode(str);
+                    return m.create(environment, tmp, repo, org + "/" + repo + "/" + contents.getPath(), tmp.sha(),
+                            GitUrl.create("ssh://gitea@" + hostname + "/" + org + "/" + repo + ".git"));
+                }
+            }
+        }
+        return null;
+    }
+}