Use go mod download as a fetcher

.github/workflows/ci.yml

@@ -17,18 +17,48 @@ jobs:
     - uses: cachix/install-nix-action@v12
     - uses: actions/checkout@v1
     - name: Check format
-      run: ./check-fmt
+      run: nix-shell --run 'nixpkgs-fmt --check .'
 
-  build:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        nixpkgs_version: ["master"]
-        # os: [ubuntu-latest, macos-latest]
+  gomod2nix_toml:
+    runs-on: ubuntu-latest
     env:
       NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-unstable.tar.gz"
     steps:
     - uses: cachix/install-nix-action@v12
     - uses: actions/checkout@v1
-    - run: nix-shell --run 'make'
+    - name: "Build gomod2nix"
+      run: nix-shell --run "go build"
+    - name: Run gomod2nix
+      run: nix-shell --run gomod2nix
+    - name: Check diff
+      run: git diff --exit-code gomod2nix.toml
+
+  list-jobs:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: cachix/install-nix-action@v17
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+      - id: set-matrix
+        run: |
+          set -euo pipefail
+          matrix="$(go run tests/run.go list | jq --raw-input --slurp -rcM '{attr: split("\n")[:-1], os: ["ubuntu-latest"]}')"
+          echo "::set-output name=matrix::$matrix"
+
+  builds:
+    needs: list-jobs
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix: ${{fromJSON(needs.list-jobs.outputs.matrix)}}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: cachix/install-nix-action@v17
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+      - name: "Build gomod2nix"
+        run: nix-shell --run "go build"
+      - name: "Run test: ${{ matrix.attr }}"
+        run: nix-shell --run "go run tests/run.go run ${{ matrix.attr }}"

builder/default.nix

@@ -1,19 +1,41 @@
 { stdenv
+, stdenvNoCC
 , runCommand
 , buildEnv
 , lib
 , fetchgit
 , removeReferencesTo
-, go
+, jq
+, cacert
+, pkgs
 }:
 let
 
   parseGoMod = import ./parser.nix;
 
   removeExpr = refs: ''remove-references-to ${lib.concatMapStrings (ref: " -t ${ref}") refs}'';
 
+  fetchGoModule =
+    { hash
+    , goPackagePath
+    , version
+    , go ? pkgs.go
+    }:
+    stdenvNoCC.mkDerivation {
+      name = "${baseNameOf goPackagePath}_${version}";
+      builder = ./fetch.sh;
+      inherit goPackagePath version;
+      nativeBuildInputs = [ go jq ];
+      outputHashMode = "recursive";
+      outputHashAlgo = null;
+      outputHash = hash;
+      SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt";
+      impureEnvVars = lib.fetchers.proxyImpureEnvVars;
+    };
+
   buildGoApplication =
     { modules
+    , go ? pkgs.go
     , src
     , pwd ? null
     , CGO_ENABLED ? "0"
@@ -47,17 +69,15 @@ let
           nativeBuildInputs = [ go ];
           json = builtins.toJSON modulesStruct;
 
-          sources = builtins.toJSON (lib.mapAttrs
-            (goPackagePath: meta:
-              let
-                src = fetchgit {
-                  inherit (meta.fetch) url sha256 rev;
-                  fetchSubmodules = true;
-                };
-                srcPath = "${src}/${meta.relPath or ""}";
-              in
-              srcPath)
-            modulesStruct);
+          sources = builtins.toJSON (
+            lib.mapAttrs
+              (goPackagePath: meta: fetchGoModule {
+                goPackagePath = meta.replaced or goPackagePath;
+                inherit (meta) version hash;
+                inherit go;
+              })
+              modulesStruct.mod
+          );
 
           passAsFile = [ "json" "sources" ];
         }

builder/fetch.sh

@@ -0,0 +1,10 @@
+source $stdenv/setup
+
+export HOME=$(mktemp -d)
+
+# Call once first outside of subshell for better error reporting
+go mod download "$goPackagePath@$version"
+
+dir=$(go mod download --json "$goPackagePath@$version" | jq -r .Dir)
+
+cp -r $dir $out

builder/parser.nix

@@ -3,7 +3,7 @@
 # in normalised form.
 
 let
-  inherit (builtins) elemAt mapAttrs split foldl' match filter typeOf;
+  inherit (builtins) elemAt mapAttrs split foldl' match filter typeOf hasAttr length;
 
   # Strip lines with comments & other junk
   stripStr = s: elemAt (split "^ *" (elemAt (split " *$" s) 0)) 2;
@@ -16,9 +16,6 @@ let
     # Strip leading tabs characters
     (lines: map (l: elemAt (match "(\t)?(.*)" l) 1) lines)
 
-    # Strip comment lines
-    (filter (l: match "[ \t]*//.*" l != null))
-
     # Filter empty lines
     (filter (l: l != ""))
   ];
@@ -40,21 +37,40 @@ let
 
       in
       {
-        data = acc.data // (
+        data = (acc.data // (
           if directive == "" && rest == ")" then { }
-          else if inDirective != null && rest == "(" then {
+          else if inDirective != null && rest == "(" && ! hasAttr inDirective acc.data then {
             ${inDirective} = { };
-          } else if inDirective != null then {
+          }
+          else if rest == "(" || rest == ")" then { }
+          else if inDirective != null then {
             ${inDirective} = acc.data.${inDirective} // { ${directive} = rest; };
-          } else {
+          } else if directive == "replace" then
+            (
+              let
+                segments = split " => " rest;
+                getSegment = elemAt segments;
+              in
+              assert length segments == 3; {
+                replace = acc.data.replace // {
+                  ${getSegment 0} = "=> ${getSegment 2}";
+                };
+              }
+            )
+          else {
             ${directive} = rest;
           }
+        )
         );
         inherit inDirective;
       })
     {
       inDirective = null;
-      data = { };
+      data = {
+        require = { };
+        replace = { };
+        exclude = { };
+      };
     }
     lines
   ).data;
@@ -98,27 +114,21 @@ let
     data // {
       replace =
         mapAttrs
-          (n: v:
+          (_: v:
             let
               m = match "=> ([^ ]+) (.+)" v;
               m2 = match "=> (.*+)" v;
             in
             if m != null then {
               goPackagePath = elemAt m 0;
-              version = parseVersion (elemAt m 1);
+              version = elemAt m 1;
             } else {
               path = elemAt m2 0;
             })
           data.replace;
     }
   );
 
-  parseRequire = data: (
-    data // {
-      require = mapAttrs (n: v: parseVersion v) data.require;
-    }
-  );
-
   splitString = sep: s: filter (t: t != [ ]) (split sep s);
 
 in
@@ -128,5 +138,4 @@ foldl' (acc: f: f acc) (splitString "\n" contents) [
   parseLines
   normaliseDirectives
   parseReplace
-  parseRequire
 ]

builder/symlink.go

@@ -9,23 +9,21 @@ import (
 	"sort"
 )
 
-type fetchInfo struct {
-	Type   string `toml:"type"`
-	URL    string `toml:"url"`
-	Rev    string `toml:"rev"`
-	Sha256 string `toml:"sha256"`
+type Package struct {
+	GoPackagePath string `json:"-"`
+	Version       string `json:"version"`
+	Hash          string `json:"hash"`
+	ReplacedPath  string `json:"replaced,omitempty"`
 }
 
-type packageT struct {
-	SumVersion string     `toml:"sumVersion"`
-	RelPath    string     `toml:"relPath,omitempty"`
-	VendorPath string     `toml:"vendorPath,omitempty"`
-	Fetch      *fetchInfo `toml:"fetch"`
+type Output struct {
+	SchemaVersion int                 `json:"schema"`
+	Mod           map[string]*Package `json:"mod"`
 }
 
 func main() {
 
-	pkgs := make(map[string]*packageT)
+	var output Output
 	sources := make(map[string]string)
 
 	b, err := ioutil.ReadFile(os.Getenv("sourcesPath"))
@@ -43,11 +41,13 @@ func main() {
 		panic(err)
 	}
 
-	err = json.Unmarshal(b, &pkgs)
+	err = json.Unmarshal(b, &output)
 	if err != nil {
 		panic(err)
 	}
 
+	pkgs := output.Mod
+
 	keys := make([]string, 0, len(pkgs))
 	for key := range pkgs {
 		keys = append(keys, key)
@@ -58,12 +58,8 @@ func main() {
 	for i := len(keys) - 1; i >= 0; i-- {
 		key := keys[i]
 		src := sources[key]
-		pkg := pkgs[key]
 
 		paths := []string{key}
-		if pkg.VendorPath != "" {
-			paths = append(paths, pkg.VendorPath)
-		}
 
 		for _, path := range paths {
 

default.nix

@@ -1,6 +1,10 @@
-{ buildGoApplication, lib, makeWrapper, nix-prefetch-git }:
+{ buildGoApplication, go_1_18, nix, lib, makeWrapper, nix-prefetch-git }:
 
+let
+  go = go_1_18;
+in
 buildGoApplication {
+  inherit go;
   pname = "gomod2nix";
   version = "0.1";
   src = lib.cleanSourceWith {
@@ -9,12 +13,14 @@ buildGoApplication {
   };
   modules = ./gomod2nix.toml;
 
+  allowGoReference = true;
+
   subPackages = [ "." ];
 
   nativeBuildInputs = [ makeWrapper ];
 
   postInstall = ''
-    wrapProgram $out/bin/gomod2nix --prefix PATH : ${lib.makeBinPath [ nix-prefetch-git ]}
+    wrapProgram $out/bin/gomod2nix --prefix PATH : ${lib.makeBinPath [ go ]}
     rm -f $out/bin/builder
   '';
 }