diff --git a/flake.nix b/flake.nix index 21cdde3a5d..67f2bc3a18 100644 --- a/flake.nix +++ b/flake.nix @@ -226,6 +226,7 @@ # a dev shell for working on dream2nix # use via 'nix develop . -c $SHELL' + # TODO only for the current system? devShells = forAllSystems (system: pkgs: let makeDevshell = import "${inp.devshell}/modules" pkgs; mkShell = config: diff --git a/overrides/nodejs/default.nix b/overrides/nodejs/default.nix index eed79ceccb..5e83e70f30 100644 --- a/overrides/nodejs/default.nix +++ b/overrides/nodejs/default.nix @@ -578,6 +578,12 @@ in }; }; + # Teach node-gyp to use nodejs headers locally + # rather that download them from https://nodejs.org. + node-gyp.build.postInstall = '' + wrapProgram "$out/bin/node-gyp" --set npm_config_nodedir $nodeSources + ''; + node-hid = { build = { nativeBuildInputs = old: @@ -589,6 +595,10 @@ in }; }; + nodegit.build.buildInputs = with pkgs; + old: + old ++ [pkg-config libkrb5 e2fsprogs]; + npm = { dont-install-deps = { installDeps = ""; @@ -642,6 +652,10 @@ in }; }; + sharp.build.buildInputs = with pkgs; + old: + old ++ [vips.dev glib.dev pkg-config]; + simple-git-hooks = { dont-postinstall = { buildScript = "true"; @@ -660,6 +674,17 @@ in }; }; + sqlite3 = { + build = { + # See its README for build instructions + # TODO It needs different flags for electron, use $electronHeaders + buildScript = '' + node-pre-gyp install --build-from-source --nodedir=$nodeSources --offline --runtime=node --sqlite=${pkgs.sqlite} + ''; + nativeBuildInputs = old: old ++ [pkgs.sqlite]; + }; + }; + tabby = { inherit cntr; fix-build = { diff --git a/src/default.nix b/src/default.nix index 50457e277f..11a365107c 100644 --- a/src/default.nix +++ b/src/default.nix @@ -284,7 +284,7 @@ in let getSourceSpec getRoot getDependencies - getCyclicDependencies + getCyclicHelpers defaultPackageName defaultPackageVersion packages diff --git a/src/subsystems/haskell/builders/default/default.nix b/src/subsystems/haskell/builders/default/default.nix index ef09ed63c2..a843f8b778 100644 --- a/src/subsystems/haskell/builders/default/default.nix +++ b/src/subsystems/haskell/builders/default/default.nix @@ -14,15 +14,22 @@ in { }: { ### FUNCTIONS # AttrSet -> Bool) -> AttrSet -> [x] - getCyclicDependencies, # name: version: -> [ {name=; version=; } ] - getDependencies, # name: version: -> [ {name=; version=; } ] - getSource, # name: version: -> store-path + # name: version: -> helpers + getCyclicHelpers, + # name: version: -> [ {name=; version=; } ] + getDependencies, + # name: version: -> store-path + getSource, # to get information about the original source spec - getSourceSpec, # name: version: -> {type="git"; url=""; hash="";} + # name: version: -> {type="git"; url=""; hash="";} + getSourceSpec, ### ATTRIBUTES - subsystemAttrs, # attrset - defaultPackageName, # string - defaultPackageVersion, # string + # attrset + subsystemAttrs, + # string + defaultPackageName, + # string + defaultPackageVersion, # all exported (top-level) package names and versions # attrset of pname -> version, packages, diff --git a/src/subsystems/nodejs/builders/granular/default.nix b/src/subsystems/nodejs/builders/granular/default.nix index 7e8c976087..96007d743a 100644 --- a/src/subsystems/nodejs/builders/granular/default.nix +++ b/src/subsystems/nodejs/builders/granular/default.nix @@ -14,15 +14,23 @@ ... }: { # Funcs - # AttrSet -> Bool) -> AttrSet -> [x] - getCyclicDependencies, # name: version: -> [ {name=; version=; } ] - getDependencies, # name: version: -> [ {name=; version=; } ] - getSource, # name: version: -> store-path + # name: version: -> helpers + getCyclicHelpers, + # name: version: -> [ {name=; version=; } ] + getDependencies, + # name: version: -> store-path + getSource, + # name: version: -> {type="git"; url=""; hash="";} + extra values from npm packages + getSourceSpec, # Attributes - subsystemAttrs, # attrset - defaultPackageName, # string - defaultPackageVersion, # string - packages, # list + # attrset + subsystemAttrs, + # string + defaultPackageName, + # string + defaultPackageVersion, + # list + packages, # attrset of pname -> versions, # where versions is a list of version strings packageVersions, @@ -37,7 +45,8 @@ b = builtins; l = lib // builtins; - nodejsVersion = subsystemAttrs.nodejsVersion; + nodejsVersion = subsystemAttrs.nodejsVersion or null; + transitiveBinaries = subsystemAttrs.transitiveBinaries or false; isMainPackage = name: version: (args.packages."${name}" or null) == version; @@ -45,14 +54,11 @@ nodejs = if args ? nodejs then args.nodejs - else + else if nodejsVersion != null + then pkgs."nodejs-${builtins.toString nodejsVersion}_x" - or (throw "Could not find nodejs version '${nodejsVersion}' in pkgs"); - - nodeSources = runCommandLocal "node-sources" {} '' - tar --no-same-owner --no-same-permissions -xf ${nodejs.src} - mv node-* $out - ''; + or (throw "Could not find nodejs version '${nodejsVersion}' in pkgs") + else pkgs.nodejs; allPackages = lib.mapAttrs @@ -104,7 +110,7 @@ local ver ver="v$(cat $electronDist/version | tr -d '\n')" mkdir $TMP/$ver - cp $electronHeaders $TMP/$ver/node-$ver-headers.tar.gz + cp --no-preserve=mode $electronHeaders $TMP/$ver/node-$ver-headers.tar.gz # calc checksums cd $TMP/$ver @@ -115,18 +121,11 @@ python -m http.server 45034 --directory $TMP & # copy electron distribution - cp -r $electronDist $TMP/electron - chmod -R +w $TMP/electron + cp -r --no-preserve=mode $electronDist $TMP/electron # configure electron toolchain - ${pkgs.jq}/bin/jq ".build.electronDist = \"$TMP/electron\"" package.json \ - | ${pkgs.moreutils}/bin/sponge package.json - - ${pkgs.jq}/bin/jq ".build.linux.target = \"dir\"" package.json \ - | ${pkgs.moreutils}/bin/sponge package.json - - ${pkgs.jq}/bin/jq ".build.npmRebuild = false" package.json \ - | ${pkgs.moreutils}/bin/sponge package.json + jq ".build.electronDist = \"$TMP/electron\" | .build.linux.target = \"dir\" | .build.npmRebuild = false" package.json > package.json.tmp + mv package.json.tmp package.json # execute electron-rebuild if available export headers=http://localhost:45034/ @@ -140,33 +139,146 @@ # Only executed for electron based packages. # Creates an executable script under /bin starting the electron app - electron-wrap = + electronBin = if pkgs.stdenv.isLinux - then '' - mkdir -p $out/bin - makeWrapper \ - $electronDist/electron \ - $out/bin/$(basename "$packageName") \ - --add-flags "$(realpath $electronAppDir)" - '' - else '' - mkdir -p $out/bin - makeWrapper \ - $electronDist/Electron.app/Contents/MacOS/Electron \ - $out/bin/$(basename "$packageName") \ - --add-flags "$(realpath $electronAppDir)" - ''; + then "$electronDist/electron" + else "$electronDist/Electron.app/Contents/MacOS/Electron"; # Generates a derivation for a specific package name + version makePackage = name: version: let - pname = lib.replaceStrings ["@" "/"] ["__at__" "__slash__"] name; - - deps = getDependencies name version; - - nodeDeps = - lib.forEach - deps - (dep: allPackages."${dep.name}"."${dep.version}"); + pname = name; + + rawDeps = getDependencies name version; + inherit (getCyclicHelpers name version) cycleeDeps cyclicParent isCyclee isThisCycleeFor replaceCyclees; + + # cycles + # for nodejs, we need to copy any cycles into a single package together + # getCyclicHelpers already cut the cycles for us, into one cyclic (e.g. eslint) and many cyclee (e.g. eslint-util) + # when a package is cyclic: + # - the cyclee deps should not be in the cyclic/node_modules folder + # - the cyclee deps need to be copied into the package next to cyclic + # so node can find them all together + # when a package is cyclee: + # - the cyclic dep should not be in the cyclee/node_modules folder + # when a dep is cyclee: + # - the dep path should point into the cyclic parent + + # Keep only the deps we can install, assume it all works out + deps = let + myOS = with stdenv.targetPlatform; + if isLinux + then "linux" + else if isDarwin + then "darwin" + else ""; + in + replaceCyclees (lib.filter + ( + dep: let + p = allPackages."${dep.name}"."${dep.version}"; + s = p.extraInfo; + in + # this dep is a cyclee + !(isCyclee dep.name dep.version) + # this dep is not for this os + && ((s.os or null == null) || lib.any (o: o == myOS) s.os) + # this package is a cyclee + && !(isThisCycleeFor dep.name dep.version) + ) + rawDeps); + + nodePkgs = + l.map + (dep: let + pkg = allPackages."${dep.name}"."${dep.version}"; + in + if dep ? replaces + then pkg // {packageName = dep.replaces.name;} + else pkg) + deps; + cycleePkgs = + l.map + (dep: allPackages."${dep.name}"."${dep.version}") + cycleeDeps; + + # Derivation building the ./node_modules directory in isolation. + makeModules = { + withDev ? false, + withOptionals ? true, + }: let + isMain = isMainPackage name version; + # These flags will only be present if true. Also, dev deps are required for non-main packages + myDeps = + lib.filter + (dep: let + s = dep.extraInfo; + in + (withOptionals || !(s.optional or false)) + && (!isMain || (withDev || !(s.dev or false)))) + nodePkgs; + in + if lib.length myDeps == 0 + then null + else + pkgs.runCommandLocal "node_modules-${pname}" {} '' + shopt -s nullglob + set -e + + mkdir $out + + function doLink() { + local name=$(basename $1) + local target="$2/$name" + if [ -e "$target" ]; then + local link=$(readlink $target) + if [ "$link" = $1 ]; then + # cyclic dep, all ok + return + fi + echo "Cannot overwrite $target (-> $link) with $1 - incorrect cycle! Versions issue?" >&2 + exit 1 + fi + ln -s $1 $target + } + + function linkDep() { + local pkg=$1 + local name=$2 + # special case for namespaced modules + if [[ $name == @* ]]; then + local namespace=$(dirname $name) + mkdir -p $out/$namespace + doLink $pkg/lib/node_modules/$name $out/$namespace + else + doLink $pkg/lib/node_modules/$name $out + fi + } + + ${l.toString (l.map + (d: "linkDep ${l.toString d} ${d.packageName}\n") + myDeps)} + + # symlink module executables to ./node_modules/.bin + mkdir $out/.bin + for dep in ${l.toString myDeps}; do + # We assume dotfiles are not public binaries + for b in $dep/bin/*; do + if [ -L "$b" ]; then + # when these relative symlinks, make absolute + # last one wins (-sf) + ln -sf $(readlink -f $b) $out/.bin/$(basename $b) + else + # e.g. wrapped binary + ln -sf $b $out/.bin/$(basename $b) + fi + done + done + # remove empty .bin + rmdir $out/.bin || true + ''; + prodModules = makeModules {withDev = false;}; + # if noDev was used, these are just the prod modules + devModules = makeModules {withDev = true;}; passthruDeps = l.listToAttrs @@ -203,13 +315,22 @@ then null else pkgs."electron_${electronVersionMajor}".headers; + hasExtraInfo = subsystemAttrs ? extraInfo; + extraInfo = subsystemAttrs.extraInfo.${name}.${version} or {}; + # If the translator doesn't provide extraInfo, assume scripts + hasInstall = + if hasExtraInfo + then extraInfo.hasInstallScript or false + else true; + isMain = isMainPackage name version; + pkg = produceDerivation name (stdenv.mkDerivation rec { inherit dependenciesJson electronHeaders - nodeDeps - nodeSources version + transitiveBinaries + prodModules ; packageName = name; @@ -219,52 +340,63 @@ passthru.dependencies = passthruDeps; passthru.devShell = import ./devShell.nix { - inherit - mkShell - nodejs - packageName - pkg - ; + inherit mkShell nodejs devModules; }; + passthru.extraInfo = extraInfo; + /* For top-level packages install dependencies as full copies, as this reduces errors with build tooling that doesn't cope well with symlinking. */ - installMethod = - if isMainPackage name version - then "copy" - else "symlink"; + # TODO implement copy and make configurable + # installMethod = + # if isMain + # then "copy" + # else "symlink"; electronAppDir = "."; # only run build on the main package - runBuild = isMainPackage name version; + runBuild = isMain && (subsystemAttrs.hasBuildScript or true); + + # can be overridden to define alternative install command + # (defaults to npm install steps) + buildScript = null; + shouldBuild = hasInstall || runBuild || buildScript != null || electronHeaders != null; + buildModules = + if runBuild + then devModules + else prodModules; + nodeSources = + if shouldBuild + then nodejs + else null; - src = getSource name version; + # We don't need unpacked sources + src = let t = getSource name version; in t.original or t; - nativeBuildInputs = [makeWrapper]; + nativeBuildInputs = + if shouldBuild + then [makeWrapper] + else []; - buildInputs = [jq nodejs python3]; + # We must provide nodejs even when not building to allow + # patchShebangs to find it for binaries + buildInputs = + if shouldBuild || (!hasExtraInfo || (extraInfo ? bin)) + then [jq nodejs python3] + else [python3]; # prevents running into ulimits - passAsFile = ["dependenciesJson" "nodeDeps"]; + passAsFile = ["dependenciesJson"]; preConfigurePhases = ["d2nLoadFuncsPhase" "d2nPatchPhase"]; - # can be overridden to define alternative install command - # (defaults to 'npm run postinstall') - buildScript = null; - # python script to modify some metadata to support installation # (see comments below on d2nPatchPhase) fixPackage = "${./fix-package.py}"; - - # script to install (symlink or copy) dependencies. - installDeps = "${./install-deps.py}"; - - # python script to link bin entries from package.json linkBins = "${./link-bins.py}"; # costs performance and doesn't seem beneficial in most scenarios @@ -282,25 +414,17 @@ continue fi echo "copying $f" - chmod +wx $(dirname "$f") - mv "$f" "$f.bak" - mkdir "$f" - if [ -n "$(ls -A "$f.bak/")" ]; then - cp -r "$f.bak"/* "$f/" - chmod -R +w $f - fi - rm "$f.bak" + l=$(readlink -f $f) + rm -f "$f" + cp -r --no-preserve=mode "$l" "$f" done } ''; - # TODO: upstream fix to nixpkgs + # https://github.com/NixOS/nixpkgs/pull/50961#issuecomment-449638192 # example which requires this: # https://registry.npmjs.org/react-window-infinite-loader/-/react-window-infinite-loader-1.0.7.tgz - unpackCmd = - if lib.hasSuffix ".tgz" src - then "tar --delay-directory-restore -xf $src" - else null; + TAR_OPTIONS = "--delay-directory-restore"; unpackPhase = '' runHook preUnpack @@ -328,9 +452,8 @@ # Figure out what directory has been unpacked export packageDir="$(find . -maxdepth 1 -type d | tail -1)" - # Restore write permissions - find "$packageDir" -type d -exec chmod u+x {} \; - chmod -R u+w -- "$packageDir" + # Ensure write + directory execute permissions + chmod -R u+w,a+X -- "$packageDir" # Move the extracted tarball into the output folder mv -- "$packageDir" "$sourceRoot" @@ -338,37 +461,38 @@ then export strippedName="$(stripHash $src)" - # Restore write permissions - chmod -R u+w -- "$strippedName" + # Ensure write + directory execute permissions + chmod -R u+w,a+X -- "$strippedName" # Move the extracted directory into the output folder mv -- "$strippedName" "$sourceRoot" fi + # provide bin, we'll remove it if unused + mkdir $out/bin + # We keep the binaries in /bin but node uses .bin + # Symlink so that wrapper scripts etc work + ln -s ../../bin $nodeModules/.bin + runHook postUnpack ''; # The python script wich is executed in this phase: # - ensures that the package is compatible to the current system + # (if not already filtered above with os prop from translator) # - ensures the main version in package.json matches the expected # - pins dependency versions in package.json # (some npm commands might otherwise trigger networking) # - creates symlinks for executables declared in package.json + # - Any usage of 'link:' in deps will be replaced with the exact version # Apart from that: - # - Any usage of 'link:' in package.json is replaced with 'file:' # - If package-lock.json exists, it is deleted, as it might conflict # with the parent package-lock.json. d2nPatchPhase = '' # delete package-lock.json as it can lead to conflicts rm -f package-lock.json - # repair 'link:' -> 'file:' - mv $nodeModules/$packageName/package.json $nodeModules/$packageName/package.json.old - cat $nodeModules/$packageName/package.json.old | sed 's!link:!file\:!g' > $nodeModules/$packageName/package.json - rm $nodeModules/$packageName/package.json.old - # run python script (see comment above): - cp package.json package.json.bak python $fixPackage \ || \ # exit code 3 -> the package is incompatible to the current platform @@ -380,48 +504,44 @@ else exit 1 fi - - # configure typescript - if [ -f ./tsconfig.json ] \ - && node -e 'require("typescript")' &>/dev/null; then - node ${./tsconfig-to-json.js} - ${pkgs.jq}/bin/jq ".compilerOptions.preserveSymlinks = true" tsconfig.json \ - | ${pkgs.moreutils}/bin/sponge tsconfig.json - fi ''; - # - installs dependencies into the node_modules directory - # - adds executables of direct node module dependencies to PATH - # - adds the current node module to NODE_PATH + # - links dependencies into the node_modules directory + adds bin to PATH # - sets HOME=$TMPDIR, as this is required by some npm scripts - # TODO: don't install dev dependencies. Load into NODE_PATH instead configurePhase = '' runHook preConfigure - # symlink sub dependencies as well as this imitates npm better - python $installDeps - - echo "Symlinking transitive executables to $nodeModules/.bin" - for dep in ${l.toString nodeDeps}; do - binDir=$dep/lib/node_modules/.bin - if [ -e $binDir ]; then - for bin in $(ls $binDir/); do - mkdir -p $nodeModules/.bin - - # symlink might have been already created by install-deps.py - # if installMethod=copy was selected - if [ ! -e $nodeModules/.bin/$bin ]; then - ln -s $binDir/$bin $nodeModules/.bin/$bin + if [ -n "$buildModules" ]; then + if [ -L $sourceRoot/node_modules ] || [ -e $sourceRoot/node_modules ]; then + echo Warning: The source $sourceRoot includes a node_modules directory. Replacing. >&2 + rm -rf $sourceRoot/node_modules + fi + ln -s $buildModules $sourceRoot/node_modules + if [ -d $buildModules/.bin ]; then + export PATH="$PATH:$sourceRoot/node_modules/.bin" + fi + fi + ${ + # Here we copy cyclee deps into the cyclehead node_modules + # so the cyclic deps can find each other + if cycleePkgs != [] + then '' + for dep in ${l.toString cycleePkgs}; do + # We must copy everything so Node finds it + # Let's hope that clashing names are just duplicates + # keep write perms with no-preserve + cp -rf --no-preserve=mode $dep/lib/node_modules/* $nodeModules + if [ -d $dep/bin ]; then + # this copies symlinks as-is, so they will point to the + # local target when relative, and module-local links + # are made relative by nixpkgs post-build + # last one wins (-f) + cp -af --no-preserve=mode $dep/bin/. $out/bin/. fi done - fi - done - - # add bin path entries collected by python script - export PATH="$PATH:$nodeModules/.bin" - - # add dependencies to NODE_PATH - export NODE_PATH="$NODE_PATH:$nodeModules/$packageName/node_modules" + '' + else "" + } export HOME=$TMPDIR @@ -430,47 +550,77 @@ # Runs the install command which defaults to 'npm run postinstall'. # Allows using custom install command by overriding 'buildScript'. - buildPhase = '' - runHook preBuild + # TODO this logic supposes a build script, which is not documented + # for installing, we only need to run `npm run install` (pre and post scripts run automatically) + # https://github.com/npm/npm/issues/5919 + # TODO build first if has build, give it devModules during build + + buildPhase = + if shouldBuild + then '' + set -x + runHook preBuild + + if [ -n "$shouldBuild" ]; then + # execute electron-rebuild + if [ -n "$electronHeaders" ]; then + echo "executing electron-rebuild" + ${electron-rebuild} + fi - # execute electron-rebuild - if [ -n "$electronHeaders" ]; then - echo "executing electron-rebuild" - ${electron-rebuild} - fi + # execute install command + if [ -n "$buildScript" ]; then + if [ -f "$buildScript" ]; then + $buildScript + else + eval "$buildScript" + fi + else + if [ -n "$runBuild" ]; then + # by default, only for top level packages, `npm run build` is executed + npm run --if-present build + fi - # execute install command - if [ -n "$buildScript" ]; then - if [ -f "$buildScript" ]; then - $buildScript - else - eval "$buildScript" - fi - # by default, only for top level packages, `npm run build` is executed - elif [ -n "$runBuild" ] && [ "$(jq '.scripts.build' ./package.json)" != "null" ]; then - npm run build - else - if [ "$(jq '.scripts.preinstall' ./package.json)" != "null" ]; then - npm --production --offline --nodedir=$nodeSources run preinstall - fi - if [ "$(jq '.scripts.install' ./package.json)" != "null" ]; then - npm --production --offline --nodedir=$nodeSources run install - fi - if [ "$(jq '.scripts.postinstall' ./package.json)" != "null" ]; then - npm --production --offline --nodedir=$nodeSources run postinstall + # This seems to be the only script that needs running on install + npm --omit=dev --offline --nodedir=$nodeSources run --if-present install + fi fi - fi - runHook postBuild - ''; + runHook postBuild + set +x + '' + else "true"; # Symlinks executables and manual pages to correct directories installPhase = '' runHook preInstall + if [ "$buildModules" != "$prodModules" ]; then + if [ -n "$prodModules" ]; then + ln -sf $prodModules $sourceRoot/node_modules + else + rm $sourceRoot/node_modules + fi + fi + echo "Symlinking bin entries from package.json" python $linkBins + if [ -n "$transitiveBinaries" ]; then + # pass down transitive binaries, like npm does + # all links are absolute so we can just copy + cp -af --no-preserve=mode $prodModules/.bin/. $out/bin/. + fi + + if rmdir $out/bin 2>/dev/null; then + # we didn't install any binaries + rm $nodeModules/.bin + else + # make sure binaries are executable, following symlinks + # ignore failures from symlinks pointing to other pkgs + chmod a+x $out/bin/* 2>/dev/null || true + fi + echo "Symlinking manual pages" if [ -d "$nodeModules/$packageName/man" ] then @@ -488,7 +638,11 @@ # wrap electron app if [ -n "$electronHeaders" ]; then echo "Wrapping electron app" - ${electron-wrap} + mkdir -p $out/bin + makeWrapper \ + ${electronBin} \ + $out/bin/$(basename "$packageName") \ + --add-flags "$(realpath $electronAppDir)" fi runHook postInstall diff --git a/src/subsystems/nodejs/builders/granular/devShell.nix b/src/subsystems/nodejs/builders/granular/devShell.nix index 82b69c556b..b1b92e509d 100644 --- a/src/subsystems/nodejs/builders/granular/devShell.nix +++ b/src/subsystems/nodejs/builders/granular/devShell.nix @@ -17,51 +17,23 @@ with a fully reproducible copy again. { mkShell, nodejs, - packageName, - pkg, + devModules, }: mkShell { - buildInputs = [ - nodejs - ]; - shellHook = let - /* - This uses the existig package derivation, and modifies it, to - disable all phases but the one which creates the ./node_modules. - - The result is a derivation only generating the node_modules and - .bin directories. - - TODO: This is be a bit hacky and could be abstracted better - TODO: Don't always delete all of ./node_modules. Only overwrite - missing or changed modules. - */ - nodeModulesDrv = pkg.overrideAttrs (old: { - buildPhase = ":"; - installMethod = "copy"; - dontPatch = true; - dontBuild = true; - dontInstall = true; - dontFixup = true; - # the configurePhase fails if these variables are not set - d2nPatchPhase = '' - nodeModules=$out/lib/node_modules - mkdir -p $nodeModules/$packageName - cd $nodeModules/$packageName - ''; - }); - nodeModulesDir = "${nodeModulesDrv}/lib/node_modules/${packageName}/node_modules"; - binDir = "${nodeModulesDrv}/lib/node_modules/.bin"; - in '' - # create the ./node_modules directory - rm -rf ./node_modules - mkdir -p ./node_modules/.bin - cp -r ${nodeModulesDir}/* ./node_modules/ - for link in $(ls ${binDir}); do - target=$(readlink ${binDir}/$link | cut -d'/' -f4-) - ln -s ../$target ./node_modules/.bin/$link - done - chmod -R +w ./node_modules - export PATH="$PATH:$(realpath ./node_modules)/.bin" - ''; + buildInputs = [nodejs]; + # TODO implement copy, maybe + shellHook = + if devModules != null + then '' + # create the ./node_modules directory + if [ -e ./node_modules ] && [ ! -L ./node_modules ]; then + echo -e "\nFailed creating the ./node_modules symlink to '${devModules}'" + echo -e "\n./node_modules already exists and is a directory, which means it is managed by another program. Please delete ./node_modules first and re-enter the dev shell." + else + rm -f ./node_modules + ln -s ${devModules} ./node_modules + export PATH="$PATH:$(realpath ./node_modules)/.bin" + fi + '' + else ""; } diff --git a/src/subsystems/nodejs/builders/granular/fix-package.py b/src/subsystems/nodejs/builders/granular/fix-package.py index 9c60b4b06f..d67c5e4c1c 100644 --- a/src/subsystems/nodejs/builders/granular/fix-package.py +++ b/src/subsystems/nodejs/builders/granular/fix-package.py @@ -14,7 +14,7 @@ changed = False -# fail if platform incompatible +# fail if platform incompatible - should not happen due to filters if 'os' in package_json: platform = sys.platform if platform not in package_json['os']\ @@ -39,7 +39,7 @@ f"{package_json.get('version')} -> {version}", file=sys.stderr ) - changed = True + package_json['origVersion'] = package_json['version'] package_json['version'] = version @@ -48,6 +48,7 @@ # as NPM install will otherwise re-fetch these if 'dependencies' in package_json: dependencies = package_json['dependencies'] + depsChanged = False # dependencies can be a list or dict for pname in dependencies: if 'bundledDependencies' in package_json\ @@ -58,17 +59,21 @@ f"WARNING: Dependency {pname} wanted but not available. Ignoring.", file=sys.stderr ) + depsChanged = True continue version =\ 'unknown' if isinstance(dependencies, list) else dependencies[pname] if available_deps[pname] != version: - version = available_deps[pname] - changed = True + depsChanged = True print( f"package.json: Pinning version '{version}' to '{available_deps[pname]}'" f" for dependency '{pname}'", file=sys.stderr ) + if depsChanged: + changed = True + package_json['dependencies'] = available_deps + package_json['origDependencies'] = dependencies # write changes to package.json if changed: diff --git a/src/subsystems/nodejs/builders/granular/install-deps.py b/src/subsystems/nodejs/builders/granular/install-deps.py deleted file mode 100644 index 9d74e09924..0000000000 --- a/src/subsystems/nodejs/builders/granular/install-deps.py +++ /dev/null @@ -1,212 +0,0 @@ -import json -import os -import pathlib -import shutil -import subprocess as sp -import sys - - -pname = os.environ.get('packageName') -version = os.environ.get('version') -bin_dir = f"{os.path.abspath('..')}/.bin" -root = f"{os.path.abspath('.')}/node_modules" -package_json_cache = {} - - -with open(os.environ.get("nodeDepsPath")) as f: - nodeDeps = f.read().split() - -def get_package_json(path): - if path not in package_json_cache: - if not os.path.isfile(f"{path}/package.json"): - return None - with open(f"{path}/package.json") as f: - package_json_cache[path] = json.load(f) - return package_json_cache[path] - -def install_direct_dependencies(): - if not os.path.isdir(root): - os.mkdir(root) - with open(os.environ.get('nodeDepsPath')) as f: - deps = f.read().split() - for dep in deps: - if os.path.isdir(f"{dep}/lib/node_modules"): - for module in os.listdir(f"{dep}/lib/node_modules"): - # ignore hidden directories - if module[0] == ".": - continue - if module[0] == '@': - for submodule in os.listdir(f"{dep}/lib/node_modules/{module}"): - pathlib.Path(f"{root}/{module}").mkdir(exist_ok=True) - print(f"installing: {module}/{submodule}") - origin =\ - os.path.realpath(f"{dep}/lib/node_modules/{module}/{submodule}") - if not os.path.exists(f"{root}/{module}/{submodule}"): - os.symlink(origin, f"{root}/{module}/{submodule}") - else: - print(f"installing: {module}") - origin = os.path.realpath(f"{dep}/lib/node_modules/{module}") - if not os.path.isdir(f"{root}/{module}"): - os.symlink(origin, f"{root}/{module}") - else: - print(f"already exists: {root}/{module}") - - -def collect_dependencies(root, depth): - if not os.path.isdir(root): - return [] - dirs = os.listdir(root) - - currentDeps = [] - for d in dirs: - if d.rpartition('/')[-1].startswith('@'): - subdirs = os.listdir(f"{root}/{d}") - for sd in subdirs: - cur_dir = f"{root}/{d}/{sd}" - currentDeps.append(f"{cur_dir}") - else: - cur_dir = f"{root}/{d}" - currentDeps.append(cur_dir) - - if depth == 0: - return currentDeps - else: - depsOfDeps =\ - map(lambda dep: collect_dependencies(f"{dep}/node_modules", depth - 1), currentDeps) - result = [] - for deps in depsOfDeps: - result += deps - return result - - -def symlink_sub_dependencies(): - for dep in collect_dependencies(root, 1): - # compute module path - d1, d2 = dep.split('/')[-2:] - if d1.startswith('@'): - path = f"{root}/{d1}/{d2}" - else: - path = f"{root}/{d2}" - - # check for collision - if os.path.isdir(path): - continue - - # create parent dir - pathlib.Path(os.path.dirname(path)).mkdir(parents=True, exist_ok=True) - - # symlink dependency - os.symlink(os.path.realpath(dep), path) - - -# create symlinks for executables (bin entries from package.json) -def symlink_bin(bin_dir, package_location, package_json, force=False): - if package_json and 'bin' in package_json and package_json['bin']: - bin = package_json['bin'] - - def link(name, relpath): - source = f'{bin_dir}/{name}' - sourceDir = os.path.dirname(source) - # create parent dir - pathlib.Path(sourceDir).mkdir(parents=True, exist_ok=True) - dest = os.path.relpath(f'{package_location}/{relpath}', sourceDir) - print(f"symlinking executable. dest: {dest}; source: {source}") - if force and os.path.exists(source): - os.remove(source) - if not os.path.exists(source): - os.symlink(dest, source) - - if isinstance(bin, str): - name = package_json['name'].split('/')[-1] - link(name, bin) - - else: - for name, relpath in bin.items(): - link(name, relpath) - - -# checks if dependency is already installed in the current or parent dir. -def dependency_satisfied(root, pname, version): - if root == "/nix/store": - return False - - parent = os.path.dirname(root) - - if os.path.isdir(f"{root}/{pname}"): - package_json_file = f"{root}/{pname}/package.json" - if os.path.isfile(package_json_file): - if version == get_package_json(f"{root}/{pname}").get('version'): - return True - - return dependency_satisfied(parent, pname, version) - - -# transforms symlinked dependencies into real copies -def symlinks_to_copies(node_modules): - sp.run(f"chmod +wx {node_modules}".split()) - for dep in collect_dependencies(node_modules, 0): - - # only handle symlinks to directories - if not os.path.islink(dep) or os.path.isfile(dep): - continue - - d1, d2 = dep.split('/')[-2:] - if d1[0] == '@': - pname = f"{d1}/{d2}" - sp.run(f"chmod +wx {node_modules}/{d1}".split()) - else: - pname = d2 - - package_json = get_package_json(dep) - if package_json is not None: - version = package_json['version'] - if dependency_satisfied(os.path.dirname(node_modules), pname, version): - os.remove(dep) - continue - - print(f"copying {dep}") - os.rename(dep, f"{dep}.bac") - os.mkdir(dep) - contents = os.listdir(f"{dep}.bac") - if contents != []: - for node in contents: - if os.path.isdir(f"{dep}.bac/{node}"): - shutil.copytree(f"{dep}.bac/{node}", f"{dep}/{node}", symlinks=True) - if os.path.isdir(f"{dep}/node_modules"): - symlinks_to_copies(f"{dep}/node_modules") - else: - shutil.copy(f"{dep}.bac/{node}", f"{dep}/{node}") - os.remove(f"{dep}.bac") - symlink_bin(f"{bin_dir}", dep, package_json) - - -def symlink_direct_bins(): - deps = [] - package_json_file = get_package_json(f"{os.path.abspath('.')}") - - if package_json_file: - if 'devDependencies' in package_json_file and package_json_file['devDependencies']: - for dep,_ in package_json_file['devDependencies'].items(): - deps.append(dep) - if 'dependencies' in package_json_file and package_json_file['dependencies']: - for dep,_ in package_json_file['dependencies'].items(): - deps.append(dep) - - for name in deps: - package_location = f"{root}/{name}" - package_json = get_package_json(package_location) - symlink_bin(f"{bin_dir}", package_location, package_json, force=True) - - -# install direct deps -install_direct_dependencies() - -# symlink non-colliding deps -symlink_sub_dependencies() - -# symlinks to copies -if os.environ.get('installMethod') == 'copy': - symlinks_to_copies(root) - -# symlink direct deps bins -symlink_direct_bins() diff --git a/src/subsystems/nodejs/builders/granular/link-bins.py b/src/subsystems/nodejs/builders/granular/link-bins.py index 26c28629d9..7008871163 100644 --- a/src/subsystems/nodejs/builders/granular/link-bins.py +++ b/src/subsystems/nodejs/builders/granular/link-bins.py @@ -35,7 +35,5 @@ def link(name, relpath): for name, relpath in bin.items(): link(name, relpath) -# symlink current packages executables to $nodeModules/.bin -symlink_bin(f'{out}/lib/node_modules/.bin/', package_json) # symlink current packages executables to $out/bin symlink_bin(f'{out}/bin/', package_json) diff --git a/src/subsystems/nodejs/builders/granular/tsconfig-to-json.js b/src/subsystems/nodejs/builders/granular/tsconfig-to-json.js deleted file mode 100644 index 1ccd3db042..0000000000 --- a/src/subsystems/nodejs/builders/granular/tsconfig-to-json.js +++ /dev/null @@ -1,19 +0,0 @@ -try { - console.log(require.resolve("typescript")); -} catch(e) { - console.error("typescript is not found"); - process.exit(e.code); -} - -const ts = require("typescript") -const fs = require('fs') - -try { - const data = fs.readFileSync('tsconfig.json', 'utf8') -} catch (err) { - console.error(err) -} - -config = ts.parseConfigFileTextToJson(data) -newdata = JSON.stringify(config) -fs.writeFileSync('tsconfig.json', newdata); diff --git a/src/subsystems/nodejs/discoverers/default/default.nix b/src/subsystems/nodejs/discoverers/default/default.nix index a2fe5bc90b..589a9bddcb 100644 --- a/src/subsystems/nodejs/discoverers/default/default.nix +++ b/src/subsystems/nodejs/discoverers/default/default.nix @@ -35,9 +35,10 @@ in childrenRemoved; - getTranslatorNames = path: let - nodes = l.readDir path; - packageJson = l.fromJSON (l.readFile "${path}/package.json"); + getTranslatorNames = subTree: let + # TODO use nodejsUtils.getWorkspaceLockFile + packageJson = subTree.files."package.json".jsonContent; + lockJson = subTree.files."package-lock.json".jsonContent or null; translators = # if the package has no dependencies we use the # package-lock translator with `packageLock = null` @@ -46,17 +47,21 @@ && (packageJson.devDependencies or {} == {}) && (packageJson.workspaces or [] == []) then ["package-lock"] + else if lockJson != null + then let + lockVersion = lockJson.lockfileVersion or 0; + in + if lockVersion == 1 + then ["package-lock"] + else if lockVersion == 2 || lockVersion == 3 + then ["package-lock-v2"] + else ["package-json"] else - l.optionals (nodes ? "package-lock.json") ["package-lock"] - ++ l.optionals (nodes ? "yarn.lock") ["yarn-lock"] + l.optionals (subTree.files ? "yarn.lock") ["yarn-lock"] ++ ["package-json"]; in translators; - # returns the parsed package.json of a given directory - getPackageJson = dirPath: - l.fromJSON (l.readFile "${dirPath}/package.json"); - # returns all relative paths to workspaces defined by a glob getWorkspacePaths = glob: tree: if l.hasSuffix "*" glob @@ -121,15 +126,13 @@ makeWorkspaceProjectInfo = tree: wsRelPath: parentInfo: dlib.construct.discoveredProject { inherit subsystem; - name = - (getPackageJson "${tree.fullPath}/${wsRelPath}").name - or "${parentInfo.name}/${wsRelPath}"; + name = (tree.getNodeFromPath wsRelPath).files."package.json".jsonContent.name or "${parentInfo.name}/${wsRelPath}"; relPath = dlib.sanitizeRelativePath "${tree.relPath}/${wsRelPath}"; translators = l.unique ( - (lib.filter (trans: l.elem trans ["package-lock" "yarn-lock"]) parentInfo.translators) - ++ (getTranslatorNames "${tree.fullPath}/${wsRelPath}") + (lib.filter (trans: l.elem trans ["package-lock" "package-lock-v2" "yarn-lock"]) parentInfo.translators) + ++ (getTranslatorNames (tree.getNodeFromPath wsRelPath)) ); subsystemInfo = { workspaceParent = tree.relPath; @@ -152,7 +155,7 @@ }) (tree.directories or {})); in - # skip if not a nodajs project + # skip if not a nodejs project if alreadyDiscovered ? "${tree.relPath}" @@ -165,8 +168,14 @@ currentProjectInfo = dlib.construct.discoveredProject { inherit subsystem; inherit (tree) relPath; - name = tree.files."package.json".jsonContent.name or tree.relPath; - translators = getTranslatorNames tree.fullPath; + name = + tree.files."package.json".jsonContent.name + or ( + if tree.relPath == "" + then "noname" + else tree.relPath + ); + translators = getTranslatorNames tree; subsystemInfo = l.optionalAttrs (workspaces != []) { workspaces = l.map diff --git a/src/subsystems/nodejs/translators/package-json/default.nix b/src/subsystems/nodejs/translators/package-json/default.nix index d8e1eaae2b..cf0d83c06e 100644 --- a/src/subsystems/nodejs/translators/package-json/default.nix +++ b/src/subsystems/nodejs/translators/package-json/default.nix @@ -31,7 +31,7 @@ openssh ] '' - # accroding to the spec, the translator reads the input from a json file + # according to the spec, the translator reads the input from a json file jsonInput=$1 # read the json input @@ -40,6 +40,8 @@ relPath=$(jq '.project.relPath' -c -r $jsonInput) npmArgs=$(jq '.project.subsystemInfo.npmArgs' -c -r $jsonInput) + # TODO: Do we really need to copy everything? Just package.json + .npmrc + # is enough, no? And then pass the lock file to translate separately? cp -r $source/* ./ chmod -R +w ./ newSource=$(pwd) @@ -47,7 +49,8 @@ cd ./$relPath rm -rf package-lock.json yarn.lock - echo "translating in temp dir: $(pwd)" + echo "Translating with npm in temp dir: $(pwd)" + echo "You can avoid this by adding your own package-lock.json file" if [ "$(jq '.project.subsystemInfo.noDev' -c -r $jsonInput)" == "true" ]; then echo "excluding dev dependencies" @@ -61,7 +64,7 @@ jq ".source = \"$newSource\"" -c -r $jsonInput > $TMPDIR/newJsonInput cd $WORKDIR - ${subsystems.nodejs.translators.package-lock.translateBin} $TMPDIR/newJsonInput + ${subsystems.nodejs.translators.package-lock-v2.translateBin} $TMPDIR/newJsonInput ''; # inherit options from package-lock translator diff --git a/src/subsystems/nodejs/translators/package-lock-v2/default.nix b/src/subsystems/nodejs/translators/package-lock-v2/default.nix new file mode 100644 index 0000000000..48c34bab02 --- /dev/null +++ b/src/subsystems/nodejs/translators/package-lock-v2/default.nix @@ -0,0 +1,174 @@ +# TODO use translate2 +# TODO use package.json for v1 lock files +{ + dlib, + lib, + ... +}: let + b = builtins; + l = lib // builtins; + nodejsUtils = import ../utils.nix {inherit lib;}; + + translate = { + translatorName, + utils, + pkgs, + ... + }: { + project, + source, + tree, + # translator args + # name + # nodejs + ... + } @ args: let + b = builtins; + + name = + if (args.name or "{automatic}") != "{automatic}" + then args.name + else project.name; + tree = args.tree.getNodeFromPath project.relPath; + relPath = project.relPath; + source = "${args.source}/${relPath}"; + workspaces = project.subsystemInfo.workspaces or []; + + getResolved = tree: project: let + lock = + (nodejsUtils.getWorkspaceLockFile tree project "package-lock.json").jsonContent; + resolved = import ./v2-parse.nix {inherit lib lock source;}; + in + resolved; + + resolved = getResolved args.tree project; + + packageVersion = resolved.self.version or "unknown"; + + rootDependencies = resolved.self.deps; + + identifyGitSource = dep: + # TODO: when integrity is there, and git url is github then use tarball instead + # ! (dep ? integrity) && + dlib.identifyGitUrl dep.url; + + getVersion = dep: dep.version; + + getPath = dep: + lib.removePrefix "file:" dep.url; + + getSource = { + url, + hash, + ... + }: {inherit url hash;}; + + # TODO check that this works with workspaces + extraInfo = b.foldl' (acc: dep: + if dep.extra != {} + then l.recursiveUpdate acc {${dep.pname}.${dep.version} = dep.extra;} + else acc) {} + resolved.allDeps; + + # TODO workspaces + hasBuildScript = let + pkgJson = + (nodejsUtils.getWorkspaceLockFile tree project "package.json").jsonContent; + in + (pkgJson.scripts or {}) ? build; + in + utils.simpleTranslate + ({ + getDepByNameVer, + dependenciesByOriginalID, + ... + }: rec { + inherit translatorName; + location = relPath; + + # values + inputData = resolved.allDeps; + + defaultPackage = name; + + packages = + {"${defaultPackage}" = packageVersion;} + // (nodejsUtils.getWorkspacePackages tree workspaces); + + mainPackageDependencies = resolved.self.deps; + + subsystemName = "nodejs"; + + subsystemAttrs = { + inherit extraInfo hasBuildScript; + nodejsVersion = args.nodejs; + }; + + # functions + serializePackages = inputData: inputData; + + getName = dep: dep.pname; + + inherit getVersion; + + # TODO handle npm link maybe? not sure what it looks like in lock + getSourceType = dep: + if lib.hasPrefix "file:" dep.url + then "path" + else if identifyGitSource dep + then "git" + else "http"; + + sourceConstructors = { + git = dep: + (getSource dep) + // (dlib.parseGitUrl dep.url); + + http = dep: (getSource dep); + + path = dep: (dlib.construct.pathSource { + path = getPath dep; + rootName = project.name; + rootVersion = packageVersion; + }); + }; + + getDependencies = dep: dep.deps; + }); +in rec { + version = 2; + + type = "pure"; + + inherit translate; + + extraArgs = { + name = { + description = "The name of the main package"; + examples = [ + "react" + "@babel/code-frame" + ]; + default = "{automatic}"; + type = "argument"; + }; + + transitiveBinaries = { + description = "Should all the binaries from all modules be available, or only those from dependencies"; + default = false; + type = "boolean"; + }; + + # TODO: this should either be removed or only used to select + # the nodejs version for translating, not for building. + nodejs = { + description = "nodejs version to use for building"; + default = "16"; + examples = [ + "14" + "16" + ]; + type = "argument"; + }; + }; +} diff --git a/src/subsystems/nodejs/translators/package-lock-v2/v2-parse.nix b/src/subsystems/nodejs/translators/package-lock-v2/v2-parse.nix new file mode 100644 index 0000000000..27a2a9f17d --- /dev/null +++ b/src/subsystems/nodejs/translators/package-lock-v2/v2-parse.nix @@ -0,0 +1,106 @@ +# This parses a v2 package-lock.json file. This format includes all information +# to get correct dependencies, including peer dependencies and multiple +# versions. lock.packages is a set that includes the path of each dep, and +# this function teases it apart to know exactly which dep is being resolved. +# The format of the lockfile is documented at +# https://docs.npmjs.com/cli/v8/configuring-npm/package-lock-json/ +{ + lib, + lock, + source, +}: +assert (lib.elem lock.lockfileVersion [2 3]); let + b = builtins; + # { "node_modules/@foo/bar/node_modules/meep": pkg; ... } + pkgs = lock.packages; + lockName = lock.name or "unnamed"; + lockVersion = lock.version or "unknown"; + + # First part is always "" and path doesn't start with / + toPath = parts: let + joined = b.concatStringsSep "/node_modules/" parts; + len = b.stringLength joined; + sliced = b.substring 1 len joined; + in + sliced; + toParts = path: b.filter b.isString (b.split "/?node_modules/" path); + + getDep = name: parts: + if b.length parts == 0 + then null + else pkgs.${toPath (parts ++ [name])} or (getDep name (lib.init parts)); + resolveDep = name: parts: isOptional: let + dep = getDep name parts; + in + if dep == null + then + if !isOptional + then b.abort "Cannot resolve dependency ${name} from ${parts}" + else null + else { + inherit name; + inherit (dep) version; + }; + resolveDeps = nameSet: parts: isOptional: + if nameSet == null + then [] + else let + depNames = b.attrNames nameSet; + resolved = b.map (n: resolveDep n parts isOptional) depNames; + in + b.filter (d: d != null) resolved; + + mapPkg = path: let + parts = toParts path; + pname = let + n = lib.last parts; + in + if n == "" + then lockName + else n; + + extraAttrs = { + # platforms this package works on + os = 1; + # this is a dev dependency + dev = 1; + # this is an optional dependency + optional = 1; + # this is an optional dev dependency + devOptional = 1; + # set of binary scripts { name = relativePath } + bin = 1; # pkg needs to run install scripts + hasInstallScript = 1; + }; + getExtra = pkg: b.intersectAttrs extraAttrs pkg; + in + { + version ? "unknown", + # URL to content - only main package is not defined + resolved ? "file://${source}", + # hash for content + integrity ? null, + dependencies ? null, + devDependencies ? null, + peerDependencies ? null, + optionalDependencies ? null, + peerDependenciesMeta ? null, + ... + } @ pkg: let + deps = + lib.unique + ((resolveDeps dependencies parts false) + ++ (resolveDeps devDependencies parts true) + ++ (resolveDeps optionalDependencies parts true) + ++ (resolveDeps peerDependencies parts true) + ++ (resolveDeps peerDependenciesMeta parts true)); + in { + inherit pname version deps; + url = resolved; + hash = integrity; + extra = getExtra pkg; + }; + + allDeps = lib.mapAttrsToList mapPkg pkgs; + self = lib.findFirst (d: d.pname == lockName && d.version == lockVersion) (b.abort "Could not find main package") allDeps; +in {inherit allDeps self;} diff --git a/src/subsystems/nodejs/translators/package-lock/default.nix b/src/subsystems/nodejs/translators/package-lock/default.nix index db270907dd..8013eb914e 100644 --- a/src/subsystems/nodejs/translators/package-lock/default.nix +++ b/src/subsystems/nodejs/translators/package-lock/default.nix @@ -291,7 +291,7 @@ in rec { # the nodejs version for translating, not for building. nodejs = { description = "nodejs version to use for building"; - default = "14"; + default = null; examples = [ "14" "16" diff --git a/src/subsystems/nodejs/translators/yarn-lock/default.nix b/src/subsystems/nodejs/translators/yarn-lock/default.nix index 11c1ee50bd..d2c4db2863 100644 --- a/src/subsystems/nodejs/translators/yarn-lock/default.nix +++ b/src/subsystems/nodejs/translators/yarn-lock/default.nix @@ -389,7 +389,7 @@ in { nodejs = { description = "nodejs version to use for building"; - default = "14"; + default = null; examples = [ "14" "16" diff --git a/src/subsystems/rust/builders/build-rust-package/default.nix b/src/subsystems/rust/builders/build-rust-package/default.nix index 7ad90c891d..6b2a6ff278 100644 --- a/src/subsystems/rust/builders/build-rust-package/default.nix +++ b/src/subsystems/rust/builders/build-rust-package/default.nix @@ -9,7 +9,7 @@ subsystemAttrs, defaultPackageName, defaultPackageVersion, - getCyclicDependencies, + getCyclicHelpers, getDependencies, getSource, getSourceSpec, diff --git a/src/subsystems/rust/builders/crane/default.nix b/src/subsystems/rust/builders/crane/default.nix index a708c84c0c..cfd577dd99 100644 --- a/src/subsystems/rust/builders/crane/default.nix +++ b/src/subsystems/rust/builders/crane/default.nix @@ -9,7 +9,7 @@ subsystemAttrs, defaultPackageName, defaultPackageVersion, - getCyclicDependencies, + getCyclicHelpers, getDependencies, getSource, getSourceSpec, @@ -27,7 +27,8 @@ then externals.crane toolchain else if toolchain ? cargo then - externals.crane { + externals.crane + { cargoHostTarget = toolchain.cargo; cargoBuildBuild = toolchain.cargo; } diff --git a/src/subsystems/translators.nix b/src/subsystems/translators.nix index af191bca35..3b3930f815 100644 --- a/src/subsystems/translators.nix +++ b/src/subsystems/translators.nix @@ -54,7 +54,6 @@ dreamLock = dreamLock'.result or dreamLock'; in dream2nix.utils.dreamLock.toJSON - # don't use nix to detect cycles, this will be more efficient in python (dreamLock // { _generic = builtins.removeAttrs dreamLock._generic [ \"cyclicDependencies\" ]; }) diff --git a/src/templates/builders/default.nix b/src/templates/builders/default.nix index cadb984f86..befd3c01e1 100644 --- a/src/templates/builders/default.nix +++ b/src/templates/builders/default.nix @@ -11,15 +11,22 @@ }: { ### FUNCTIONS # AttrSet -> Bool) -> AttrSet -> [x] - getCyclicDependencies, # name: version: -> [ {name=; version=; } ] - getDependencies, # name: version: -> [ {name=; version=; } ] - getSource, # name: version: -> store-path + # name: version: -> helpers + getCyclicHelpers, + # name: version: -> [ {name=; version=; } ] + getDependencies, + # name: version: -> store-path + getSource, # to get information about the original source spec - getSourceSpec, # name: version: -> {type="git"; url=""; hash="";} + # name: version: -> {type="git"; url=""; hash="";} + getSourceSpec, ### ATTRIBUTES - subsystemAttrs, # attrset - defaultPackageName, # string - defaultPackageVersion, # string + # attrset + subsystemAttrs, + # string + defaultPackageName, + # string + defaultPackageVersion, # all exported (top-level) package names and versions # attrset of pname -> version, packages, diff --git a/src/utils/dream-lock.nix b/src/utils/dream-lock.nix index e766527fb7..dad20634b2 100644 --- a/src/utils/dream-lock.nix +++ b/src/utils/dream-lock.nix @@ -136,9 +136,67 @@ (dep: ! b.elem dep cyclicDependencies."${pname}"."${version}" or []) dependencyGraph."${pname}"."${version}" or []; - getCyclicDependencies = pname: version: - cyclicDependencies."${pname}"."${version}" or []; + # inverted cyclicDependencies { name.version = parent } + cyclicParents = with l; + foldAttrs (c: acc: acc // (listToAttrs [(nameValuePair c.version c.cyclic)])) {} (flatten (mapAttrsToList (cyclicName: cyclicVersions: + mapAttrsToList (cyclicVersion: cycleeDeps: + map (cycleeDep: (listToAttrs [ + ( + nameValuePair cycleeDep.name + { + version = cycleeDep.version; + cyclic = { + name = cyclicName; + version = cyclicVersion; + }; + } + ) + ])) + cycleeDeps) + cyclicVersions) + cyclicDependencies)); + + getCyclicHelpers = name: version: let + # [ {name; version} ] + cycleeDeps = cyclicDependencies."${name}"."${version}" or []; + + # {name: {version: true}} + cycleeMap = lib.foldAttrs (depVersion: acc: + acc + // (lib.listToAttrs [ + { + name = depVersion; + value = true; + } + ])) {} + cycleeDeps; + + cyclicParent = cyclicParents."${name}"."${version}" or null; + isCyclee = depName: depVersion: cycleeMap."${depName}"."${depVersion}" or false; + isThisCycleeFor = depName: depVersion: + cyclicParent + == { + name = depName; + version = depVersion; + }; + replaceCyclees = deps: + with l; + filter (d: d != null) + (map (d: let + parent = cyclicParents."${d.name}"."${d.version}" or null; + in + if parent != null + then + if parent == cyclicParent + # These packages will be part of their parent package + then null + else parent // {replaces = d;} + else d) + deps); + in { + inherit cycleeDeps cyclicParent isCyclee isThisCycleeFor replaceCyclees; + }; getRoot = pname: version: let spec = getSourceSpec pname version; in @@ -157,7 +215,7 @@ defaultPackageName defaultPackageVersion subsystemAttrs - getCyclicDependencies + getCyclicHelpers getDependencies getSourceSpec getRoot