Merge pull request #12 from lorentey/restore-linux-support

Fix Linux compatibility

Package.swift

@@ -24,8 +24,7 @@ let package = Package(
     .target(name: "_AtomicsShims"),
     .target(
       name: "Atomics",
-      dependencies: ["_AtomicsShims"],
-      path: "Sources/Atomics"
+      dependencies: ["_AtomicsShims"]
     ),
     .testTarget(
       name: "AtomicsTests",

README.md

@@ -79,6 +79,20 @@ Of particular note is full support for atomic strong references. This provides a
 
 All atomic operations exposed by this package are guaranteed to have lock-free implementations. However, we do not guarantee wait-free operation -- depending on the capabilities of the target platform, some of the exposed operations may be implemented by compare-and-exchange loops. That said, all atomic operations map directly to dedicated CPU instructions where available -- to the extent supported by llvm & Clang.
 
+## Portability Concerns
+
+Lock-free double-wide atomics requires support for such things from the underlying target platform. Where such support isn't available, this package doesn't implement `DoubleWord` atomics or atomic strong references. While modern multiprocessing CPUs have been providing double-wide atomic instructions for a number of years now, some platforms still target older architectures by default; these require a special compiler option to enable double-wide atomic instructions. This currently includes Linux operating systems running on x86_64 processors, where the `cmpxchg16b` instruction isn't considered a baseline requirement.
+
+To enable double-wide atomics on Linux/x86_64, you currently have to manually supply a couple of additional options on the SPM build invocation:
+
+```
+$ swift build -Xcc -mcx16 -Xswiftc -DENABLE_DOUBLEWIDE_ATOMICS -c release
+```
+
+(`-mcx16` turns on support for `cmpxchg16b` in Clang, and `-DENABLE_DOUBLEWIDE_ATOMICS` makes Swift aware that double-wide atomics are available. Note that the resulting binaries won't run on some older AMD64 CPUs.)
+
+The package cannot currently configure this automatically.
+
 ## Memory Management
 
 Atomic access is implemented in terms of dedicated atomic storage representations that are kept distinct from the corresponding regular (non-atomic) type. (E.g., the actual integer value underlying the counter above isn't directly accessible.) This has several advantages:

Sources/Atomics/AtomicStrongReference.swift

@@ -12,6 +12,10 @@
 
 import _AtomicsShims
 
+// Double-wide atomic primitives on x86_64 CPUs aren't available by default
+// on Linux distributions, and we cannot currently enable them automatically.
+#if !(os(Linux) && arch(x86_64)) || ENABLE_DOUBLEWIDE_ATOMICS
+
 /// A class type that supports atomic strong references.
 public protocol AtomicReference: AnyObject, AtomicOptionalWrappable
 where
@@ -569,3 +573,4 @@ extension AtomicOptionalReferenceStorage: AtomicStorage {
   }
 }
 
+#endif // ENABLE_DOUBLEWIDE_ATOMICS

Sources/Atomics/IntegerConformances.swift.gyb

@@ -24,6 +24,11 @@ ${autogenerated_warning()}
 import _AtomicsShims
 
 % for swiftType in atomicTypes():
+%  if swiftType == "DoubleWord":
+// Double-wide atomic primitives on x86_64 CPUs aren't available by default
+// on Linux distributions, and we cannot currently enable them automatically.
+#if !(os(Linux) && arch(x86_64)) || ENABLE_DOUBLEWIDE_ATOMICS
+%  end
 extension ${swiftType}: AtomicValue {
   public struct AtomicRepresentation {
     public typealias Value = ${swiftType}
@@ -219,4 +224,7 @@ extension ${swiftType}.AtomicRepresentation: AtomicIntegerStorage {
 }
 % end
 
+%  if swiftType == "DoubleWord":
+#endif // ENABLE_DOUBLEWIDE_ATOMICS
+%  end
 % end

Sources/Atomics/autogenerated/IntegerConformances.swift

@@ -4939,6 +4939,9 @@ extension UInt8.AtomicRepresentation: AtomicIntegerStorage {
   }
 }
 
+// Double-wide atomic primitives on x86_64 CPUs aren't available by default
+// on Linux distributions, and we cannot currently enable them automatically.
+#if !(os(Linux) && arch(x86_64)) || ENABLE_DOUBLEWIDE_ATOMICS
 extension DoubleWord: AtomicValue {
   public struct AtomicRepresentation {
     public typealias Value = DoubleWord
@@ -5262,3 +5265,4 @@ extension DoubleWord.AtomicRepresentation: AtomicStorage {
 }
 
 
+#endif // ENABLE_DOUBLEWIDE_ATOMICS

Sources/_AtomicsShims/include/_AtomicsShims.h

@@ -59,6 +59,20 @@
 #include <stdatomic.h>
 #include <assert.h>
 
+// For now, assume double-wide atomics are available everywhere,
+// except on Linux/x86_64, where they need to be manually enabled
+// by the `cx16` target attribute. (Unfortunately we cannot currently
+// turn that on in our package description.)
+#ifdef __APPLE__
+#  define ENABLE_DOUBLEWIDE_ATOMICS 1
+#elif defined(_WIN32)
+#  define ENABLE_DOUBLEWIDE_ATOMICS 1
+#elif defined(__linux__)
+#  if !defined(__x86_64__) || defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16)
+#    define ENABLE_DOUBLEWIDE_ATOMICS 1
+#  endif
+#endif
+
 #define SWIFTATOMIC_INLINE static inline __attribute__((__always_inline__))
 #define SWIFTATOMIC_SWIFT_NAME(name) __attribute__((swift_name(#name)))
 
@@ -320,9 +334,13 @@ _sa_dword _sa_decode_dword(__uint128_t value) {
   return result;
 }
 
+#if ENABLE_DOUBLEWIDE_ATOMICS
 #define SWIFTATOMIC_ENCODE_DoubleWord(_value) (_value).value
 #define SWIFTATOMIC_DECODE_DoubleWord(value) _sa_decode_dword(value)
 SWIFTATOMIC_DEFINE_TYPE(COMPLEX, DoubleWord, _sa_dword, __uint128_t)
+#else
+SWIFTATOMIC_STORAGE_TYPE(DoubleWord, _sa_dword, __uint128_t)
+#endif
 
 #elif __INTPTR_WIDTH__ == 32
 
@@ -352,15 +370,21 @@ _sa_dword _sa_decode_dword(uint64_t value) {
   return result;
 }
 
+#if ENABLE_DOUBLEWIDE_ATOMICS
 #define SWIFTATOMIC_ENCODE_DoubleWord(_value) (_value).value
 #define SWIFTATOMIC_DECODE_DoubleWord(value) _sa_decode_dword(value)
 SWIFTATOMIC_DEFINE_TYPE(COMPLEX, DoubleWord, _sa_dword, uint64_t)
+#else
+SWIFTATOMIC_STORAGE_TYPE(DoubleWord, _sa_dword, uint64_t)
+#endif // ENABLE_DOUBLEWIDE_ATOMICS
 
 #else
 #error "Unsupported intptr_t bit width"
 #endif // __INTPTR_WIDTH
 
+#if ENABLE_DOUBLEWIDE_ATOMICS
 extern void _sa_retain_n(void *object, uint32_t n);
 extern void _sa_release_n(void *object, uint32_t n);
+#endif
 
 #endif //SWIFTATOMIC_HEADER_INCLUDED

Sources/_AtomicsShims/src/_AtomicsShims.c

@@ -10,14 +10,23 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <dlfcn.h>
 #include "_AtomicsShims.h"
 
-void _sa_retain_n(void *object, uint32_t n) {
+#if ENABLE_DOUBLEWIDE_ATOMICS
+// FIXME: These should be static inline header-only shims, but Swift 5.3 doesn't
+// like calls to swift_retain_n/swift_release_n appearing in Swift code, not
+// even when imported through C. (See https://bugs.swift.org/browse/SR-13708)
+
+void _sa_retain_n(void *object, uint32_t n)
+{
   extern void *swift_retain_n(void *object, uint32_t n);
   swift_retain_n(object, n);
 }
 
-void _sa_release_n(void *object, uint32_t n) {
+void _sa_release_n(void *object, uint32_t n)
+{
   extern void swift_release_n(void *object, uint32_t n);
   swift_release_n(object, n);
 }
+#endif

Tests/AtomicsTests/Basics.swift.gyb

@@ -28,22 +28,26 @@
     ("UInt64",     "UInt64", "12", "23"),
 
     ("Bool",       "Bool",   "true", "false"),
-    ("DoubleWord", "DoubleWord", "DoubleWord(high: 100, low: 64)", "DoubleWord(high: 50, low: 32)"),
 
-    ("Pointer",                "UnsafePointer<Foo>",         "_foo1",  "_foo2"),
-    ("OptionalPointer",        "UnsafePointer<Foo>?",        "nil",    "_foo2"),
-    ("MutablePointer",         "UnsafeMutablePointer<Foo>",  "_mfoo1", "_mfoo2"),
-    ("OptionalMutablePointer", "UnsafeMutablePointer<Foo>?", "nil",    "_mfoo2"),
-    ("RawPointer",             "UnsafeMutableRawPointer",    "_raw1",  "_raw2"),
-    ("OptionalRawPointer",     "UnsafeMutableRawPointer?",   "nil",    "_raw2"),
+    ("Pointer",                   "UnsafePointer<Foo>",         "_foo1",  "_foo2"),
+    ("OptionalPointer",           "UnsafePointer<Foo>?",        "nil",    "_foo2"),
+    ("MutablePointer",            "UnsafeMutablePointer<Foo>",  "_mfoo1", "_mfoo2"),
+    ("OptionalMutablePointer",    "UnsafeMutablePointer<Foo>?", "nil",    "_mfoo2"),
+
+    ("RawPointer",                "UnsafeRawPointer",           "_raw1",  "_raw2"),
+    ("OptionalRawPointer",        "UnsafeRawPointer?",          "nil",    "_raw2"),
+    ("MutableRawPointer",         "UnsafeMutableRawPointer",    "_mraw1", "_mraw2"),
+    ("OptionalMutableRawPointer", "UnsafeMutableRawPointer?",   "nil",    "_mraw2"),
 
-    ("Unmanaged",              "Unmanaged<Bar>",             "_bar1", "_bar2"),
-    ("OptionalUnmanaged",      "Unmanaged<Bar>?",            "nil",   "_bar2"),
+    ("Unmanaged",                 "Unmanaged<Bar>",             "_bar1", "_bar2"),
+    ("OptionalUnmanaged",         "Unmanaged<Bar>?",            "nil",   "_bar2"),
 
-    ("Reference",              "Baz",  "_baz1", "_baz2"),
-    ("OptionalReference",      "Baz?", "nil",   "_baz2"),
+    ("RawRepresentable",          "Fred", "Fred.one", "Fred.two"),
+
+    ("DoubleWord", "DoubleWord", "DoubleWord(high: 100, low: 64)", "DoubleWord(high: 50, low: 32)"),
 
-    ("RawRepresentable", "Fred", "Fred.one", "Fred.two"),
+    ("Reference",                 "Baz",  "_baz1", "_baz2"),
+    ("OptionalReference",         "Baz?", "nil",   "_baz2"),
   ]
 }%
 ${autogenerated_warning()}
@@ -72,6 +76,7 @@ private class Bar: Equatable, CustomStringConvertible {
   }
 }
 
+#if !(os(Linux) && arch(x86_64)) || ENABLE_DOUBLEWIDE_ATOMICS
 private class Baz: Equatable, CustomStringConvertible, AtomicReference {
   var value: Int
   init(_ value: Int) { self.value = value }
@@ -80,21 +85,28 @@ private class Baz: Equatable, CustomStringConvertible, AtomicReference {
     left === right
   }
 }
+#endif
 
 private enum Fred: Int, AtomicValue {
   case one
   case two
 }
 
+% for label, type, a, b in types:
+% if label == "DoubleWord" or label == "Reference" or label == "OptionalReference":
+#if !(os(Linux) && arch(x86_64)) || ENABLE_DOUBLEWIDE_ATOMICS
+% else:
+#if true
+% end
 /// Exercises all operations in a single-threaded context, verifying
 /// they provide the expected results.
-class BasicAtomicTests: XCTestCase {
+class BasicAtomic${label}Tests: XCTestCase {
+% if label == "Pointer" or label == "OptionalPointer" or label == "MutablePointer" or label == "OptionalMutablePointer":
   private let _mfoo1: UnsafeMutablePointer<Foo> = {
     let p = UnsafeMutablePointer<Foo>.allocate(capacity: 1)
     p.initialize(to: Foo(1))
     return p
   }()
-
   private let _mfoo2: UnsafeMutablePointer<Foo> = {
     let p = UnsafeMutablePointer<Foo>.allocate(capacity: 1)
     p.initialize(to: Foo(2))
@@ -104,29 +116,38 @@ class BasicAtomicTests: XCTestCase {
   private var _foo1: UnsafePointer<Foo> { UnsafePointer(_mfoo1) }
   private var _foo2: UnsafePointer<Foo> { UnsafePointer(_mfoo2) }
 
-  private let _raw1 = UnsafeMutableRawPointer.allocate(byteCount: 16, alignment: 1)
-  private let _raw2 = UnsafeMutableRawPointer.allocate(byteCount: 16, alignment: 1)
-  private let _bar1 = Unmanaged<Bar>.passRetained(Bar(1))
-  private let _bar2 = Unmanaged<Bar>.passRetained(Bar(2))
-  private let _baz1 = Baz(1)
-  private let _baz2 = Baz(2)
-
   deinit {
     _mfoo1.deinitialize(count: 1)
     _mfoo1.deallocate()
 
     _mfoo2.deinitialize(count: 1)
     _mfoo2.deallocate()
+  }
+% elif label == "RawPointer" or label == "OptionalRawPointer" or label == "MutableRawPointer" or label == "OptionalMutableRawPointer":
+  private let _mraw1 = UnsafeMutableRawPointer.allocate(byteCount: 16, alignment: 1)
+  private let _mraw2 = UnsafeMutableRawPointer.allocate(byteCount: 16, alignment: 1)
 
-    _raw1.deallocate()
-    _raw2.deallocate()
+  private var _raw1: UnsafeRawPointer { UnsafeRawPointer(_mraw1) }
+  private var _raw2: UnsafeRawPointer { UnsafeRawPointer(_mraw2) }
 
+  deinit {
+    _mraw1.deallocate()
+    _mraw2.deallocate()
+  }
+% elif label == "Unmanaged" or label == "OptionalUnmanaged":
+  private let _bar1 = Unmanaged<Bar>.passRetained(Bar(1))
+  private let _bar2 = Unmanaged<Bar>.passRetained(Bar(2))
+
+  deinit {
     _bar1.release()
     _bar2.release()
   }
+% elif label == "Reference" or label == "OptionalReference":
+  private let _baz1 = Baz(1)
+  private let _baz2 = Baz(2)
+% end
 
-% for label, type, a, b in types:
-  func test_${label}_create_destroy() {
+  func test_create_destroy() {
     let v = UnsafeAtomic<${type}>.create(${a})
     defer { v.destroy() }
     XCTAssertEqual(v.load(ordering: .relaxed), ${a})
@@ -137,7 +158,7 @@ class BasicAtomicTests: XCTestCase {
   }
 
 % for (order, _) in loadOrderings:
-  func test_${label}_load_${order}() {
+  func test_load_${order}() {
     let v = UnsafeAtomic<${type}>.create(${a})
     defer { v.destroy() }
     XCTAssertEqual(v.load(ordering: .${order}), ${a})
@@ -149,7 +170,7 @@ class BasicAtomicTests: XCTestCase {
 % end
 
 % for (order, _) in storeOrderings:
-  func test_${label}_store_${order}() {
+  func test_store_${order}() {
     let v = UnsafeAtomic<${type}>.create(${a})
     defer { v.destroy() }
     v.store(${b}, ordering: .${order})
@@ -163,7 +184,7 @@ class BasicAtomicTests: XCTestCase {
 % end
 
 % for (order, _, _) in updateOrderings:
-  func test_${label}_exchange_${order}() {
+  func test_exchange_${order}() {
     let v = UnsafeAtomic<${type}>.create(${a})
     defer { v.destroy() }
 
@@ -179,7 +200,7 @@ class BasicAtomicTests: XCTestCase {
 % end
 
 % for (order, _, _) in updateOrderings:
-  func test_${label}_compareExchange_${order}() {
+  func test_compareExchange_${order}() {
     let v = UnsafeAtomic<${type}>.create(${a})
     defer { v.destroy() }
 
@@ -220,7 +241,7 @@ class BasicAtomicTests: XCTestCase {
 % for operation in ["compareExchange", "weakCompareExchange"]:
 %   for (successorder, _, _) in updateOrderings:
 %     for (failorder, _) in loadOrderings:
-  func test_${label}_${operation}_${successorder}_${failorder}() {
+  func test_${operation}_${successorder}_${failorder}() {
     let v = UnsafeAtomic<${type}>.create(${a})
     defer { v.destroy() }
 
@@ -265,12 +286,11 @@ class BasicAtomicTests: XCTestCase {
 % end
 
 
-// Bool operations
-
 % if type == "Bool":
+  // Bool operations
 %   for (name, _, operator, arglabel, _) in boolOperations:
 %     for (order, _, _) in updateOrderings:
-  func test_${label}_loadThen${name}_${order}() {
+  func test_loadThen${name}_${order}() {
     let v = UnsafeAtomic<Bool>.create(false)
     defer { v.destroy() }
 
@@ -292,7 +312,7 @@ class BasicAtomicTests: XCTestCase {
 
 %   for (name, _, operator, arglabel, _) in boolOperations:
 %     for (order, _, _) in updateOrderings:
-  func test_${label}_${lowerFirst(name)}ThenLoad_${order}() {
+  func test_${lowerFirst(name)}ThenLoad_${order}() {
     let v = UnsafeAtomic<Bool>.create(false)
     defer { v.destroy() }
 
@@ -315,12 +335,11 @@ class BasicAtomicTests: XCTestCase {
 %   end
 % end
 
-// Integer operations
-
 % if type.startswith("Int") or type.startswith("UInt"):
+// Integer operations
 %   for (name, _, operator, arglabel, _) in integerOperations:
 %     for (order, _, _) in updateOrderings:
-  func test_${label}_loadThen${name}_${order}() {
+  func test_loadThen${name}_${order}() {
     let a: ${type} = 3
     let b: ${type} = 8
     let c: ${type} = 12
@@ -343,7 +362,7 @@ class BasicAtomicTests: XCTestCase {
 
 %   for (name, _, operator, arglabel, _) in integerOperations:
 %     for (order, _, _) in updateOrderings:
-  func test_${label}_${lowerFirst(name)}ThenLoad_${order}() {
+  func test_${lowerFirst(name)}ThenLoad_${order}() {
     let a: ${type} = 3
     let b: ${type} = 8
     let c: ${type} = 12
@@ -365,5 +384,6 @@ class BasicAtomicTests: XCTestCase {
 %   end
 % end
 
-% end
 }
+#endif
+% end