Fix filter prerelease snapshots

.github/workflows/main.yml

@@ -169,7 +169,7 @@ jobs:
         run: |
           curl http://localhost:52773/registry/packages/-/all | jq
           curl http://localhost:52773/registry/packages/zpm/ | jq
-          ASSET_NAME='zpm-0.7.1-beta.4.xml'
+          ASSET_NAME='zpm-0.7.2.xml'
           ASSET_URL=`wget --header "Authorization: token ${GITHUB_TOKEN}" -qO- https://api.github.com/repos/intersystems/ipm/releases | jq -r ".[].assets[] | select(.name == \"${ASSET_NAME}\") | .browser_download_url"`
           wget $ASSET_URL -O /tmp/zpm.xml
           CONTAINER=$(docker run --network zpm --rm -d ${{ steps.image.outputs.name }} ${{ steps.image.outputs.flags }})

CHANGELOG.md

@@ -51,6 +51,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - #224: When updating zpm, existing configuration won't be reset
 - #482: Reenabled deployed code support without impact on embedded source control (reworks HSIEO-9277)
 - #487: When loading a package, relative paths staring with prefix "http" won't be mistaken for git repo
+- #544: When installing a package from remote repo, IPM specifies `includePrerelease` and `includeSnapshots` in HTTP request. Correctly-behaving zpm registry should respect that.
+- #557: When comparing semver against semver expressions, exclude prereleases and snapshots from the range maximum.
+- #559: Allow treating the "w" in SemVer x.y.z-w as a post-release rather than pre-release.
 
 ### Security
 -

preload/cls/IPM/Installer.cls

@@ -104,6 +104,7 @@ ClassMethod ZPMInit(pRegistry As %String = "", pAnalyticsTrackingID As %String =
     $$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue("ColorScheme","", 0))
     $$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue("PipCaller", "", 0))
     $$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue("UseStandalonePip", "", 0))
+    $$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue("SemVerPostRelease", 0, 0))
     Quit $$$OK
 }
 

src/cls/IPM/General/SemanticVersion.cls

@@ -13,8 +13,16 @@ Property Patch As %Integer(MINVAL = 0) [ Required ];
 
 Property Prerelease As %IPM.DataType.RegExString(MAXLEN = 100, REGEX = "([0-9A-Za-z-])+(\.([0-9A-Za-z-])+)*");
 
+/// This is an alias for Prerelease. It is used for code readability when SemVerPostRelease is enabled.
+Property Postrelease As %IPM.DataType.RegExString(MAXLEN = 100, REGEX = "([0-9A-Za-z-])+(\.([0-9A-Za-z-])+)*") [ Calculated, SqlComputeCode = { Set {*} = Prerelease }, SqlComputed, Transient ];
+
 Property Build As %IPM.DataType.RegExString(MAXLEN = 100, REGEX = "([0-9A-Za-z-])+(\.([0-9A-Za-z-])+)*");
 
+Method PostreleaseGet() As %IPM.DataType.RegExString
+{
+	Quit ..Prerelease
+}
+
 Method ToString() As %String [ CodeMode = expression ]
 {
 ..Major 
@@ -116,6 +124,25 @@ Method Follows(pVersion As %IPM.General.SemanticVersion) As %Boolean
 		((..Major = pVersion.Major) && (..Minor > pVersion.Minor)) ||
 		((..Major = pVersion.Major) && (..Minor = pVersion.Minor) && (..Patch > pVersion.Patch))
 
+	// Handle post-releases if enabled. With this, post-releases are considered as higher version numbers.
+	If ##class(%IPM.Repo.UniversalSettings).GetValue("SemVerPostRelease") {
+		If tFollows {
+			Return 1
+		} 
+		If (..Major < pVersion.Major) || (..Minor < pVersion.Minor) || (..Patch < pVersion.Patch) {
+			Return 0
+		} 
+		// If it reaches here, major, minor, and patch are equal. We need to check post-releases.
+		If (..Postrelease = "") {
+			Return 0
+		} ElseIf (pVersion.Postrelease = "") {
+			Return 1
+		} 	
+		// If it reaches here, major, minor, and patch are equal. Both post-releases are non-empty. Compare them.
+		Return ..CompareDotSeparatedStrings(..Postrelease, pVersion.Postrelease)
+
+	}
+
 	// Handle prereleases - messy!!
 	Set tEquals = (..Major = pVersion.Major) && (..Minor = pVersion.Minor) && (..Patch = pVersion.Patch)
 	If (..Prerelease '= "") || (pVersion.Prerelease '= "") {
@@ -135,46 +162,45 @@ Method Follows(pVersion As %IPM.General.SemanticVersion) As %Boolean
 			// We are comparing equal versions where the earlier has a prerelease.
 			Quit 1
 		} Else{
-			// Both have a prerelease, and they're different.
-			// Compare dot-separated parts of the prerelease.
-			Set tFollows = 1
-			Set tThisParts = $ListFromString(..Prerelease,".")
-			Set tOtherParts = $ListFromString(pVersion.Prerelease,".")
-
-			Set tOtherHasData = 1
-			Set tThisPointer = 0
-			Set tOtherPointer = 0
-			While $ListNext(tThisParts,tThisPointer,tThisPart) {
-				Set tOtherHasData = $ListNext(tOtherParts,tOtherPointer,tOtherPart)
-				If 'tOtherHasData {
-					// The prerelease version has more parts for this one.
-					Return 1
-				}
-				If (tOtherPart = tThisPart) {
-					// Keep looking through dot-separated parts.
-					Continue
-				}
-
-				// "Collates after" operator works nicely here.
-				// e.g., the following are true: "beta" ]] 11, 11 ]] 2, 2 ]] 1
-				If (tThisPart ]] tOtherPart) {
-					Return 1
-				} Else {
-					Return 0
-				}
-			}
-			If tFollows && tOtherHasData && $ListNext(tOtherParts,tOtherPointer,tOtherPart) {
-				// If there are still dot-separated parts left in the prerelease of the version we are comparing to,
-				// it has more than this version, and therefore this version does not follow it.
-				Quit 0
-			}
+			Quit ..CompareDotSeparatedStrings(..Prerelease, pVersion.Prerelease)
 		}
 	} ElseIf tEquals {
 		Quit (pVersion.IsSnapshot() && '..IsSnapshot())
 	}
 	Quit tFollows
 }
 
+/// Compare two dot-separated strings (usually prerelease or postrelease identifiers).
+Method CompareDotSeparatedStrings(pThis As %String, pOther As %String) As %Boolean
+{
+	Set tThisParts = $ListFromString(pThis,".")
+	Set tOtherParts = $ListFromString(pOther,".")
+
+	Set tOtherHasData = 1
+	Set tThisPointer = 0
+	Set tOtherPointer = 0
+	While $ListNext(tThisParts,tThisPointer,tThisPart) {
+		Set tOtherHasData = $ListNext(tOtherParts,tOtherPointer,tOtherPart)
+		If 'tOtherHasData {
+			// The prerelease version has more parts for this one.
+			Return 1
+		}
+		If (tOtherPart = tThisPart) {
+			// Keep looking through dot-separated parts.
+			Continue
+		}
+
+		// "Collates after" operator works nicely here.
+		// e.g., the following are true: "beta" ]] 11, 11 ]] 2, 2 ]] 1
+		If (tThisPart ]] tOtherPart) {
+			Return 1
+		} Else {
+			Return 0
+		}
+	}
+	Return 0
+}
+
 Method Satisfies(pExpression As %IPM.General.SemanticVersionExpression) As %Boolean
 {
 	Quit pExpression.IsSatisfiedBy($this)

src/cls/IPM/General/SemanticVersionExpression/Comparator.cls

@@ -93,8 +93,9 @@ Method Evaluate(pVersion As %IPM.General.SemanticVersion) As %Boolean
       Set tEquals = tEquals && (val1 = val2)
     }
   }
-	If ($Extract(..Operator,1) = "=") {
-		// Means direct equality and not <= or >= so compare pre-release as well
+	If (+##class(%IPM.Repo.UniversalSettings).GetValue("SemVerPostRelease") = 0) || ($Extract(..Operator,1) = "=")  {
+		// If SemVerPostRelease is disabled, then we should not consider post-release versions as equal to the base version
+		// ..Operator = "=" means direct equality and not <= or >= so compare pre-release as well
 		Set tEquals = tEquals && (pVersion.Prerelease = ..Prerelease)
 	}
 	If tEquals || (..Operator = "=") {

src/cls/IPM/General/SemanticVersionExpression/Range.cls

@@ -118,7 +118,7 @@ ClassMethod FromString(pRangeExpr As %String, Output pRange As %IPM.General.Sema
 					Set tMajor = $Piece(tExpr,".",1)
 					Set tMinor = $Piece(tExpr,".",2)
 
-					Set tComparators = tComparators_$ListBuild("<"_tMajor_"."_(tMinor+1)_".0")
+					Set tComparators = tComparators_$ListBuild("<"_tMajor_"."_(tMinor+1)_".0-0")
 				}
 			} ElseIf tIsCaretRange {
 				// Caret ranges:
@@ -134,7 +134,7 @@ ClassMethod FromString(pRangeExpr As %String, Output pRange As %IPM.General.Sema
 				Set tPatch = $Piece(tMajorMinorPatch,".",3)
 				If (tDotLength < 3) {
 					If (tMajor '= 0) && 'tIsXRange {
-						Set tComparators = tComparators_$ListBuild(">="_$Replace(tExpr,".x",".0"),"<"_(tMajor+1)_".0.0")
+						Set tComparators = tComparators_$ListBuild(">="_$Replace(tExpr,".x",".0"),"<"_(tMajor+1)_".0.0-0")
 						Set tIsXRange = 0
 					} Else {
 						// Detected and properly handled by X-range.
@@ -146,12 +146,12 @@ ClassMethod FromString(pRangeExpr As %String, Output pRange As %IPM.General.Sema
 					Set tMax = ""
 					If (+tMajor = 0) && (tMinor '= "x") {
 						If (+tMinor = 0) && (tPatch '= "x") {
-							Set tMax = "0.0."_(tPatch+1)
+							Set tMax = "0.0."_(tPatch+1)_"-0"
 						} Else {
-							Set tMax = "0."_(tMinor+1)_".0"
+							Set tMax = "0."_(tMinor+1)_".0-0"
 						}
 					} Else {
-						Set tMax = (tMajor+1)_".0.0"
+						Set tMax = (tMajor+1)_".0.0-0"
 					}
 
 					// Maximum
@@ -174,9 +174,9 @@ ClassMethod FromString(pRangeExpr As %String, Output pRange As %IPM.General.Sema
 					// Accept anything!
 					Set tComparators = tComparators_$ListBuild(">=0.0.0")
 				} ElseIf (tMinor = "") || (tMinor = "x") {
-					Set tComparators = tComparators_$ListBuild(">="_tMajor_".0.0","<"_(tMajor+1)_".0.0")
+					Set tComparators = tComparators_$ListBuild(">="_tMajor_".0.0","<"_(tMajor+1)_".0.0-0")
 				} ElseIf (tPatch = "") || (tPatch = "x") {
-					Set tComparators = tComparators_$ListBuild(">="_tMajor_"."_tMinor_".0","<"_tMajor_"."_(tMinor+1)_".0")
+					Set tComparators = tComparators_$ListBuild(">="_tMajor_"."_tMinor_".0","<"_tMajor_"."_(tMinor+1)_".0-0")
 				}
 			}
 		}

src/cls/IPM/Repo/Remote/PackageService.cls

@@ -50,6 +50,8 @@ Method ListModules(pSearchCriteria As %IPM.Repo.SearchCriteria) As %ListOfObject
     Set tURL = tRequest.Location_"packages/" _ name
   }
   Do tRequest.SetParam("allVersions", pSearchCriteria.AllVersions)
+  Do tRequest.SetParam("includePrerelease", pSearchCriteria.IncludePrerelease)
+  Do tRequest.SetParam("includeSnapshots", pSearchCriteria.IncludeSnapshots)
 
   Set tSC = tRequest.Get($$$URLENCODE(tURL))
 

src/cls/IPM/Repo/UniversalSettings.cls

@@ -34,7 +34,12 @@ Parameter PipCaller = "PipCaller";
 /// Indicates whether <parameter>PipCaller</parameter> is a pip executable instead of python
 Parameter UseStandalonePip = "UseStandalonePip";
 
-Parameter CONFIGURABLE = "trackingId,analytics,ColorScheme,TerminalPrompt,PublishTimeout,PipCaller,UseStandalonePip";
+/// Possible values: 0, 1
+/// Indicates whether the SemVer comparison treats the version 1.0.0-1.m1 as a post-release of 1.0.0, hence 1.0.0-1.m1 > 1.0.0
+/// Default value is 0, where 1.0.0-anystring is considered a pre-release of 1.0.0, hence 1.0.0-anystring < 1.0.0
+Parameter SemVerPostRelease = "SemVerPostRelease";
+
+Parameter CONFIGURABLE = "trackingId,analytics,ColorScheme,TerminalPrompt,PublishTimeout,PipCaller,UseStandalonePip,SemVerPostRelease";
 
 /// Returns configArray, that includes all configurable settings
 ClassMethod GetAll(Output configArray) As %Status

tests/unit_tests/Test/PM/Unit/SemVer/Abstract.cls

@@ -0,0 +1,20 @@
+Class Test.PM.Unit.SemVer.Abstract Extends %UnitTest.TestCase
+{
+
+/// Original value of the SemVerPostRelease setting
+Property OriginalSemVerPostRelease As %Boolean [ InitialExpression = {##class(%IPM.Repo.UniversalSettings).GetValue("SemVerPostRelease")} ];
+
+/// Before each test, record the original value of the SemVerPostRelease setting and set it to 0 for a clean slate
+Method OnBeforeOneTest(testname As %String) As %Status
+{
+	Set ..OriginalSemVerPostRelease = ##class(%IPM.Repo.UniversalSettings).GetValue("SemVerPostRelease")
+	Return ##class(%IPM.Repo.UniversalSettings).SetValue("SemVerPostRelease", 0)
+}
+
+/// After each test, restore the original value of the SemVerPostRelease setting
+Method OnAfterOneTest(testname As %String) As %Status
+{
+	Return ##class(%IPM.Repo.UniversalSettings).SetValue("SemVerPostRelease", ..OriginalSemVerPostRelease)
+}
+
+}

tests/unit_tests/Test/PM/Unit/SemVer/Expressions.cls

@@ -1,4 +1,4 @@
-Class Test.PM.Unit.SemVer.Expressions Extends %UnitTest.TestCase
+Class Test.PM.Unit.SemVer.Expressions Extends Test.PM.Unit.SemVer.Abstract
 {
 
 Method TestEvaluate()
@@ -29,18 +29,62 @@ Method TestEvaluate()
 
 Method TestPrerelease()
 {
+	Do ..AssertNotSatisfied(">=1.0.0", "1.0.0-1.m1")
 	Do ..AssertSatisfied("1.0.0-beta.1","1.0.0-beta.1")
 	Do ..AssertSatisfied("=1.0.0-beta.1","1.0.0-beta.1")
 	Do ..AssertSatisfied(">=1.0.0-beta.1","1.0.0-beta.1")
 	Do ..AssertSatisfied(">=1.0.0-beta.1","1.0.0-beta.2")
 	Do ..AssertSatisfied(">=1.0.0-alpha.1","1.0.0-beta.1")
-	Do ..AssertSatisfied("^1.0.0","1.0.0-1.m1")
+	Do ..AssertNotSatisfied("^1.0.0","1.0.0-1.m1") 
 	Do ..AssertNotSatisfied(">=1.0.0-beta.2","1.0.0-beta.1")
 	Do ..AssertNotSatisfied(">=1.0.0-beta.1","1.0.0-alpha.1")
 
 	// This is the problematic case without HSIEO-10520
 	Do ..AssertNotSatisfied("1.0.0-1.m1","1.0.0") 
 	Do ..AssertNotSatisfied("=1.0.0-1.m1","1.0.0")
+
+	// Issue #557: snapshots/prereleases shouldn't be considered as a lower version number
+	// Even though 1.5.0-beta.1 is technically within the 1.x range, it's a prerelease and should not be considered satisfiable.
+	// Similarly 1.1.5-beta.1 does not satisify 1.1.x. See https://github.com/npm/node-semver?tab=readme-ov-file#prerelease-tags
+	Do ..AssertNotSatisfied("1.x", "1.5.0-beta.1") 
+	Do ..AssertNotSatisfied("1.x", "2.0.0-0")
+	Do ..AssertNotSatisfied("1.x", "2.0.0-SNAPSHOT")
+	Do ..AssertNotSatisfied("1.x", "2.0.0-alpha")
+	Do ..AssertNotSatisfied("1.1.x", "1.1.5-beta.1")
+	Do ..AssertNotSatisfied("1.1.x", "1.2.0-0")
+	Do ..AssertNotSatisfied("1.1.x", "1.2.0-SNAPSHOT")
+	Do ..AssertNotSatisfied("1.1.x", "1.2.0-alpha")
+}
+
+Method TestPostRelease()
+{
+	// With SemVerPostRelease enabled, post-release versions should be considered as a higher version number
+	Set tSC = ##class(%IPM.Repo.UniversalSettings).SetValue("SemVerPostRelease", 1)
+	Do ..AssertSatisfied(">=1.0.0", "1.0.0-1.m1")
+	Do ..AssertSatisfied("1.0.0-beta.1","1.0.0-beta.1")
+	Do ..AssertSatisfied("=1.0.0-beta.1","1.0.0-beta.1")
+	Do ..AssertSatisfied(">=1.0.0-beta.1","1.0.0-beta.1")
+	Do ..AssertSatisfied(">=1.0.0-beta.1","1.0.0-beta.2")
+	Do ..AssertSatisfied(">=1.0.0-alpha.1","1.0.0-beta.1")
+
+	Do ..AssertSatisfied("^1.0.0","1.0.0-1.m1") 
+	Do ..AssertNotSatisfied(">=1.0.0-beta.2","1.0.0-beta.1")
+	Do ..AssertNotSatisfied(">=1.0.0-beta.1","1.0.0-alpha.1")
+
+	// This is the problematic case without HSIEO-10520
+	Do ..AssertNotSatisfied("1.0.0-1.m1","1.0.0") 
+	Do ..AssertNotSatisfied("=1.0.0-1.m1","1.0.0")
+
+	// With SemVerPostRelease enabled, 1.5.0-beta.1 is within the 1.x range, because it's a post-release of 1.5.0.
+	// Similarly, 1.1.5-beta.1 satisifies 1.1.x.
+	Do ..AssertSatisfied("1.x", "1.5.0-beta.1") 
+	Do ..AssertNotSatisfied("1.x", "2.0.0-0")
+	Do ..AssertNotSatisfied("1.x", "2.0.0-SNAPSHOT")
+	Do ..AssertNotSatisfied("1.x", "2.0.0-alpha")
+	Do ..AssertSatisfied("1.1.x", "1.1.5-beta.1")
+	Do ..AssertNotSatisfied("1.1.x", "1.2.0-0")
+	Do ..AssertNotSatisfied("1.1.x", "1.2.0-SNAPSHOT")
+	Do ..AssertNotSatisfied("1.1.x", "1.2.0-alpha")
 }
 
 Method TestEquivalenceHyphenRanges()
@@ -53,38 +97,38 @@ Method TestEquivalenceHyphenRanges()
 
 Method TestEquivalenceXRanges()
 {
-	Do ..AssertEquivalent("*", ">=0.0.0")
-	Do ..AssertEquivalent("", ">=0.0.0")
-	Do ..AssertEquivalent("1", ">=1.0.0 <2.0.0")
-	Do ..AssertEquivalent("1.x", ">=1.0.0 <2.0.0")
-	Do ..AssertEquivalent("1.2", ">=1.2.0 <1.3.0")
-	Do ..AssertEquivalent("1.2.*", ">=1.2.0 <1.3.0")
-	Do ..AssertEquivalent("1.2.X", ">=1.2.0 <1.3.0")
+	Do ..AssertEquivalent("*", ">=0.0.0") // Do we want to allow pre-release versions?
+	Do ..AssertEquivalent("", ">=0.0.0")  // Do we want to allow pre-release versions?
+	Do ..AssertEquivalent("1", ">=1.0.0 <2.0.0-0")
+	Do ..AssertEquivalent("1.x", ">=1.0.0 <2.0.0-0")
+	Do ..AssertEquivalent("1.2", ">=1.2.0 <1.3.0-0")
+	Do ..AssertEquivalent("1.2.*", ">=1.2.0 <1.3.0-0")
+	Do ..AssertEquivalent("1.2.X", ">=1.2.0 <1.3.0-0")
 }
 
 Method TestEquivalenceTildeRanges()
 {
-	Do ..AssertEquivalent("~1.2.3", ">=1.2.3 <1.3.0")
-	Do ..AssertEquivalent("~1.2", ">=1.2.0 <1.3.0")
-	Do ..AssertEquivalent("~1", ">=1.0.0 <2.0.0")
-	Do ..AssertEquivalent("~0.2.3", ">=0.2.3 <0.3.0")
-	Do ..AssertEquivalent("~0.2", ">=0.2.0 <0.3.0")
-	Do ..AssertEquivalent("~0", ">=0.0.0 <1.0.0")
-	Do ..AssertEquivalent("~1.2.3-beta.2", ">=1.2.3-beta.2 <1.3.0")
+	Do ..AssertEquivalent("~1.2.3", ">=1.2.3 <1.3.0-0")
+	Do ..AssertEquivalent("~1.2", ">=1.2.0 <1.3.0-0")
+	Do ..AssertEquivalent("~1", ">=1.0.0 <2.0.0-0")
+	Do ..AssertEquivalent("~0.2.3", ">=0.2.3 <0.3.0-0")
+	Do ..AssertEquivalent("~0.2", ">=0.2.0 <0.3.0-0")
+	Do ..AssertEquivalent("~0", ">=0.0.0 <1.0.0-0")
+	Do ..AssertEquivalent("~1.2.3-beta.2", ">=1.2.3-beta.2 <1.3.0-0")
 }
 
 Method TestEquivalenceCaretRanges()
 {
-	Do ..AssertEquivalent("^1.2.3", ">=1.2.3 <2.0.0")
-	Do ..AssertEquivalent("^0.2.3", ">=0.2.3 <0.3.0")
-	Do ..AssertEquivalent("^0.0.3", ">=0.0.3 <0.0.4")
-	Do ..AssertEquivalent("^1.2.3-beta.2", ">=1.2.3-beta.2 <2.0.0")
-	Do ..AssertEquivalent("^0.0.3-beta", ">=0.0.3-beta <0.0.4")
-	Do ..AssertEquivalent("^1.2.x", ">=1.2.0 <2.0.0")
-	Do ..AssertEquivalent("^0.0.x", ">=0.0.0 <0.1.0")
-	Do ..AssertEquivalent("^0.0", ">=0.0.0 <0.1.0")
-	Do ..AssertEquivalent("^1.x", ">=1.0.0 <2.0.0")
-	Do ..AssertEquivalent("^0.x", ">=0.0.0 <1.0.0")
+	Do ..AssertEquivalent("^1.2.3", ">=1.2.3 <2.0.0-0")
+	Do ..AssertEquivalent("^0.2.3", ">=0.2.3 <0.3.0-0")
+	Do ..AssertEquivalent("^0.0.3", ">=0.0.3 <0.0.4-0")
+	Do ..AssertEquivalent("^1.2.3-beta.2", ">=1.2.3-beta.2 <2.0.0-0")
+	Do ..AssertEquivalent("^0.0.3-beta", ">=0.0.3-beta <0.0.4-0")
+	Do ..AssertEquivalent("^1.2.x", ">=1.2.0 <2.0.0-0")
+	Do ..AssertEquivalent("^0.0.x", ">=0.0.0 <0.1.0-0")
+	Do ..AssertEquivalent("^0.0", ">=0.0.0 <0.1.0-0")
+	Do ..AssertEquivalent("^1.x", ">=1.0.0 <2.0.0-0")
+	Do ..AssertEquivalent("^0.x", ">=0.0.0 <1.0.0-0")
 }
 
 Method TestIRISVersions()