Statistics
| Revision:

root / tmp / org.txm.statsengine.r.core.win32 / res / win32 / library / BH / include / boost / atomic / detail / gcc-x86.hpp @ 2486

History | View | Annotate | Download (53.6 kB)

1
#ifndef BOOST_ATOMIC_DETAIL_GCC_X86_HPP
2
#define BOOST_ATOMIC_DETAIL_GCC_X86_HPP
3

    
4
//  Copyright (c) 2009 Helge Bahmann
5
//  Copyright (c) 2012 Tim Blechmann
6
//
7
//  Distributed under the Boost Software License, Version 1.0.
8
//  See accompanying file LICENSE_1_0.txt or copy at
9
//  http://www.boost.org/LICENSE_1_0.txt)
10

    
11
#include <string.h>
12
#include <cstddef>
13
#include <boost/cstdint.hpp>
14
#include <boost/atomic/detail/config.hpp>
15

    
16
#ifdef BOOST_HAS_PRAGMA_ONCE
17
#pragma once
18
#endif
19

    
20
namespace boost {
21
namespace atomics {
22
namespace detail {
23

    
24
#if defined(__x86_64__) || defined(__SSE2__)
25
# define BOOST_ATOMIC_X86_FENCE_INSTR "mfence\n"
26
#else
27
# define BOOST_ATOMIC_X86_FENCE_INSTR "lock ; addl $0, (%%esp)\n"
28
#endif
29

    
30
#define BOOST_ATOMIC_X86_PAUSE() __asm__ __volatile__ ("pause\n")
31

    
32
#if defined(__i386__) &&\
33
    (\
34
        defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8) ||\
35
        defined(__i586__) || defined(__i686__) || defined(__pentium4__) || defined(__nocona__) || defined(__core2__) || defined(__corei7__) ||\
36
        defined(__k6__) || defined(__athlon__) || defined(__k8__) || defined(__amdfam10__) || defined(__bdver1__) || defined(__bdver2__) || defined(__bdver3__) || defined(__btver1__) || defined(__btver2__)\
37
    )
38
#define BOOST_ATOMIC_X86_HAS_CMPXCHG8B 1
39
#endif
40

    
41
#if defined(__x86_64__) && defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16)
42
#define BOOST_ATOMIC_X86_HAS_CMPXCHG16B 1
43
#endif
44

    
45
inline void
46
platform_fence_before(memory_order order)
47
{
48
    switch(order)
49
    {
50
    case memory_order_relaxed:
51
    case memory_order_acquire:
52
    case memory_order_consume:
53
        break;
54
    case memory_order_release:
55
    case memory_order_acq_rel:
56
        __asm__ __volatile__ ("" ::: "memory");
57
        /* release */
58
        break;
59
    case memory_order_seq_cst:
60
        __asm__ __volatile__ ("" ::: "memory");
61
        /* seq */
62
        break;
63
    default:;
64
    }
65
}
66

    
67
inline void
68
platform_fence_after(memory_order order)
69
{
70
    switch(order)
71
    {
72
    case memory_order_relaxed:
73
    case memory_order_release:
74
        break;
75
    case memory_order_acquire:
76
    case memory_order_acq_rel:
77
        __asm__ __volatile__ ("" ::: "memory");
78
        /* acquire */
79
        break;
80
    case memory_order_consume:
81
        /* consume */
82
        break;
83
    case memory_order_seq_cst:
84
        __asm__ __volatile__ ("" ::: "memory");
85
        /* seq */
86
        break;
87
    default:;
88
    }
89
}
90

    
91
inline void
92
platform_fence_after_load(memory_order order)
93
{
94
    switch(order)
95
    {
96
    case memory_order_relaxed:
97
    case memory_order_release:
98
        break;
99
    case memory_order_acquire:
100
    case memory_order_acq_rel:
101
        __asm__ __volatile__ ("" ::: "memory");
102
        break;
103
    case memory_order_consume:
104
        break;
105
    case memory_order_seq_cst:
106
        __asm__ __volatile__ ("" ::: "memory");
107
        break;
108
    default:;
109
    }
110
}
111

    
112
inline void
113
platform_fence_before_store(memory_order order)
114
{
115
    switch(order)
116
    {
117
    case memory_order_relaxed:
118
    case memory_order_acquire:
119
    case memory_order_consume:
120
        break;
121
    case memory_order_release:
122
    case memory_order_acq_rel:
123
        __asm__ __volatile__ ("" ::: "memory");
124
        /* release */
125
        break;
126
    case memory_order_seq_cst:
127
        __asm__ __volatile__ ("" ::: "memory");
128
        /* seq */
129
        break;
130
    default:;
131
    }
132
}
133

    
134
inline void
135
platform_fence_after_store(memory_order order)
136
{
137
    switch(order)
138
    {
139
    case memory_order_relaxed:
140
    case memory_order_release:
141
        break;
142
    case memory_order_acquire:
143
    case memory_order_acq_rel:
144
        __asm__ __volatile__ ("" ::: "memory");
145
        /* acquire */
146
        break;
147
    case memory_order_consume:
148
        /* consume */
149
        break;
150
    case memory_order_seq_cst:
151
        __asm__ __volatile__ ("" ::: "memory");
152
        /* seq */
153
        break;
154
    default:;
155
    }
156
}
157

    
158
}
159
}
160

    
161
class atomic_flag
162
{
163
private:
164
    atomic_flag(const atomic_flag &) /* = delete */ ;
165
    atomic_flag & operator=(const atomic_flag &) /* = delete */ ;
166
    uint32_t v_;
167
public:
168
    BOOST_CONSTEXPR atomic_flag(void) BOOST_NOEXCEPT : v_(0) {}
169

    
170
    bool
171
    test_and_set(memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
172
    {
173
        uint32_t v = 1;
174
        atomics::detail::platform_fence_before(order);
175
        __asm__ __volatile__ (
176
            "xchgl %0, %1"
177
            : "+r" (v), "+m" (v_)
178
        );
179
        atomics::detail::platform_fence_after(order);
180
        return v;
181
    }
182

    
183
    void
184
    clear(memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
185
    {
186
        if (order == memory_order_seq_cst) {
187
            uint32_t v = 0;
188
            __asm__ __volatile__ (
189
                "xchgl %0, %1"
190
                : "+r" (v), "+m" (v_)
191
            );
192
        } else {
193
            atomics::detail::platform_fence_before(order);
194
            v_ = 0;
195
        }
196
    }
197
};
198

    
199
} /* namespace boost */
200

    
201
#define BOOST_ATOMIC_FLAG_LOCK_FREE 2
202

    
203
#include <boost/atomic/detail/base.hpp>
204

    
205
#if !defined(BOOST_ATOMIC_FORCE_FALLBACK)
206

    
207
#define BOOST_ATOMIC_CHAR_LOCK_FREE 2
208
#define BOOST_ATOMIC_CHAR16_T_LOCK_FREE 2
209
#define BOOST_ATOMIC_CHAR32_T_LOCK_FREE 2
210
#define BOOST_ATOMIC_WCHAR_T_LOCK_FREE 2
211
#define BOOST_ATOMIC_SHORT_LOCK_FREE 2
212
#define BOOST_ATOMIC_INT_LOCK_FREE 2
213
#define BOOST_ATOMIC_LONG_LOCK_FREE 2
214

    
215
#if defined(__x86_64__) || defined(BOOST_ATOMIC_X86_HAS_CMPXCHG8B)
216
#define BOOST_ATOMIC_LLONG_LOCK_FREE 2
217
#endif
218

    
219
#if defined(BOOST_ATOMIC_X86_HAS_CMPXCHG16B) && (defined(BOOST_HAS_INT128) || !defined(BOOST_NO_ALIGNMENT))
220
#define BOOST_ATOMIC_INT128_LOCK_FREE 2
221
#endif
222

    
223
#define BOOST_ATOMIC_POINTER_LOCK_FREE 2
224
#define BOOST_ATOMIC_BOOL_LOCK_FREE 2
225

    
226
namespace boost {
227

    
228
#define BOOST_ATOMIC_THREAD_FENCE 2
229
inline void
230
atomic_thread_fence(memory_order order)
231
{
232
    switch(order)
233
    {
234
    case memory_order_relaxed:
235
        break;
236
    case memory_order_release:
237
        __asm__ __volatile__ ("" ::: "memory");
238
        break;
239
    case memory_order_acquire:
240
        __asm__ __volatile__ ("" ::: "memory");
241
        break;
242
    case memory_order_acq_rel:
243
        __asm__ __volatile__ ("" ::: "memory");
244
        break;
245
    case memory_order_consume:
246
        break;
247
    case memory_order_seq_cst:
248
        __asm__ __volatile__ (BOOST_ATOMIC_X86_FENCE_INSTR ::: "memory");
249
        break;
250
    default:;
251
    }
252
}
253

    
254
#define BOOST_ATOMIC_SIGNAL_FENCE 2
255
inline void
256
atomic_signal_fence(memory_order)
257
{
258
    __asm__ __volatile__ ("" ::: "memory");
259
}
260

    
261
namespace atomics {
262
namespace detail {
263

    
264
template<typename T, bool Sign>
265
class base_atomic<T, int, 1, Sign>
266
{
267
private:
268
    typedef base_atomic this_type;
269
    typedef T value_type;
270
    typedef T difference_type;
271

    
272
protected:
273
    typedef value_type value_arg_type;
274

    
275
public:
276
    BOOST_DEFAULTED_FUNCTION(base_atomic(void), {})
277
    BOOST_CONSTEXPR explicit base_atomic(value_type v) BOOST_NOEXCEPT : v_(v) {}
278

    
279
    void
280
    store(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
281
    {
282
        if (order != memory_order_seq_cst) {
283
            platform_fence_before(order);
284
            const_cast<volatile value_type &>(v_) = v;
285
        } else {
286
            exchange(v, order);
287
        }
288
    }
289

    
290
    value_type
291
    load(memory_order order = memory_order_seq_cst) const volatile BOOST_NOEXCEPT
292
    {
293
        value_type v = const_cast<const volatile value_type &>(v_);
294
        platform_fence_after_load(order);
295
        return v;
296
    }
297

    
298
    value_type
299
    fetch_add(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
300
    {
301
        platform_fence_before(order);
302
        __asm__ __volatile__
303
        (
304
            "lock ; xaddb %0, %1"
305
            : "+q" (v), "+m" (v_)
306
            :
307
            : "cc"
308
        );
309
        platform_fence_after(order);
310
        return v;
311
    }
312

    
313
    value_type
314
    fetch_sub(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
315
    {
316
        return fetch_add(-v, order);
317
    }
318

    
319
    value_type
320
    exchange(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
321
    {
322
        platform_fence_before(order);
323
        __asm__ __volatile__
324
        (
325
            "xchgb %0, %1"
326
            : "+q" (v), "+m" (v_)
327
        );
328
        platform_fence_after(order);
329
        return v;
330
    }
331

    
332
    bool
333
    compare_exchange_strong(
334
        value_type & expected,
335
        value_type desired,
336
        memory_order success_order,
337
        memory_order failure_order) volatile BOOST_NOEXCEPT
338
    {
339
        value_type previous = expected;
340
        platform_fence_before(success_order);
341
        bool success;
342
        __asm__ __volatile__
343
        (
344
            "lock ; cmpxchgb %3, %1\n\t"
345
            "sete %2"
346
            : "+a" (previous), "+m" (v_), "=q" (success)
347
            : "q" (desired)
348
            : "cc"
349
        );
350
        if (success)
351
            platform_fence_after(success_order);
352
        else
353
            platform_fence_after(failure_order);
354
        expected = previous;
355
        return success;
356
    }
357

    
358
    bool
359
    compare_exchange_weak(
360
        value_type & expected,
361
        value_type desired,
362
        memory_order success_order,
363
        memory_order failure_order) volatile BOOST_NOEXCEPT
364
    {
365
        return compare_exchange_strong(expected, desired, success_order, failure_order);
366
    }
367

    
368
    value_type
369
    fetch_and(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
370
    {
371
        value_type tmp = load(memory_order_relaxed);
372
        while (!compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed))
373
        {
374
            BOOST_ATOMIC_X86_PAUSE();
375
        }
376
        return tmp;
377
    }
378

    
379
    value_type
380
    fetch_or(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
381
    {
382
        value_type tmp = load(memory_order_relaxed);
383
        while (!compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed))
384
        {
385
            BOOST_ATOMIC_X86_PAUSE();
386
        }
387
        return tmp;
388
    }
389

    
390
    value_type
391
    fetch_xor(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
392
    {
393
        value_type tmp = load(memory_order_relaxed);
394
        while (!compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed))
395
        {
396
            BOOST_ATOMIC_X86_PAUSE();
397
        }
398
        return tmp;
399
    }
400

    
401
    bool
402
    is_lock_free(void) const volatile BOOST_NOEXCEPT
403
    {
404
        return true;
405
    }
406

    
407
    BOOST_ATOMIC_DECLARE_INTEGRAL_OPERATORS
408

    
409
    BOOST_DELETED_FUNCTION(base_atomic(base_atomic const&))
410
    BOOST_DELETED_FUNCTION(base_atomic& operator=(base_atomic const&))
411

    
412
private:
413
    value_type v_;
414
};
415

    
416
template<typename T, bool Sign>
417
class base_atomic<T, int, 2, Sign>
418
{
419
private:
420
    typedef base_atomic this_type;
421
    typedef T value_type;
422
    typedef T difference_type;
423

    
424
protected:
425
    typedef value_type value_arg_type;
426

    
427
public:
428
    BOOST_DEFAULTED_FUNCTION(base_atomic(void), {})
429
    BOOST_CONSTEXPR explicit base_atomic(value_type v) BOOST_NOEXCEPT : v_(v) {}
430

    
431
    void
432
    store(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
433
    {
434
        if (order != memory_order_seq_cst) {
435
            platform_fence_before(order);
436
            const_cast<volatile value_type &>(v_) = v;
437
        } else {
438
            exchange(v, order);
439
        }
440
    }
441

    
442
    value_type
443
    load(memory_order order = memory_order_seq_cst) const volatile BOOST_NOEXCEPT
444
    {
445
        value_type v = const_cast<const volatile value_type &>(v_);
446
        platform_fence_after_load(order);
447
        return v;
448
    }
449

    
450
    value_type
451
    fetch_add(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
452
    {
453
        platform_fence_before(order);
454
        __asm__ __volatile__
455
        (
456
            "lock ; xaddw %0, %1"
457
            : "+q" (v), "+m" (v_)
458
            :
459
            : "cc"
460
        );
461
        platform_fence_after(order);
462
        return v;
463
    }
464

    
465
    value_type
466
    fetch_sub(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
467
    {
468
        return fetch_add(-v, order);
469
    }
470

    
471
    value_type
472
    exchange(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
473
    {
474
        platform_fence_before(order);
475
        __asm__ __volatile__
476
        (
477
            "xchgw %0, %1"
478
            : "+q" (v), "+m" (v_)
479
        );
480
        platform_fence_after(order);
481
        return v;
482
    }
483

    
484
    bool
485
    compare_exchange_strong(
486
        value_type & expected,
487
        value_type desired,
488
        memory_order success_order,
489
        memory_order failure_order) volatile BOOST_NOEXCEPT
490
    {
491
        value_type previous = expected;
492
        platform_fence_before(success_order);
493
        bool success;
494
        __asm__ __volatile__
495
        (
496
            "lock ; cmpxchgw %3, %1\n\t"
497
            "sete %2"
498
            : "+a" (previous), "+m" (v_), "=q" (success)
499
            : "q" (desired)
500
            : "cc"
501
        );
502
        if (success)
503
            platform_fence_after(success_order);
504
        else
505
            platform_fence_after(failure_order);
506
        expected = previous;
507
        return success;
508
    }
509

    
510
    bool
511
    compare_exchange_weak(
512
        value_type & expected,
513
        value_type desired,
514
        memory_order success_order,
515
        memory_order failure_order) volatile BOOST_NOEXCEPT
516
    {
517
        return compare_exchange_strong(expected, desired, success_order, failure_order);
518
    }
519

    
520
    value_type
521
    fetch_and(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
522
    {
523
        value_type tmp = load(memory_order_relaxed);
524
        while (!compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed))
525
        {
526
            BOOST_ATOMIC_X86_PAUSE();
527
        }
528
        return tmp;
529
    }
530

    
531
    value_type
532
    fetch_or(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
533
    {
534
        value_type tmp = load(memory_order_relaxed);
535
        while (!compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed))
536
        {
537
            BOOST_ATOMIC_X86_PAUSE();
538
        }
539
        return tmp;
540
    }
541

    
542
    value_type
543
    fetch_xor(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
544
    {
545
        value_type tmp = load(memory_order_relaxed);
546
        while (!compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed))
547
        {
548
            BOOST_ATOMIC_X86_PAUSE();
549
        }
550
        return tmp;
551
    }
552

    
553
    bool
554
    is_lock_free(void) const volatile BOOST_NOEXCEPT
555
    {
556
        return true;
557
    }
558

    
559
    BOOST_ATOMIC_DECLARE_INTEGRAL_OPERATORS
560

    
561
    BOOST_DELETED_FUNCTION(base_atomic(base_atomic const&))
562
    BOOST_DELETED_FUNCTION(base_atomic& operator=(base_atomic const&))
563

    
564
private:
565
    value_type v_;
566
};
567

    
568
template<typename T, bool Sign>
569
class base_atomic<T, int, 4, Sign>
570
{
571
private:
572
    typedef base_atomic this_type;
573
    typedef T value_type;
574
    typedef T difference_type;
575

    
576
protected:
577
    typedef value_type value_arg_type;
578

    
579
public:
580
    BOOST_DEFAULTED_FUNCTION(base_atomic(void), {})
581
    BOOST_CONSTEXPR explicit base_atomic(value_type v) BOOST_NOEXCEPT : v_(v) {}
582

    
583
    void
584
    store(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
585
    {
586
        if (order != memory_order_seq_cst) {
587
            platform_fence_before(order);
588
            const_cast<volatile value_type &>(v_) = v;
589
        } else {
590
            exchange(v, order);
591
        }
592
    }
593

    
594
    value_type
595
    load(memory_order order = memory_order_seq_cst) const volatile BOOST_NOEXCEPT
596
    {
597
        value_type v = const_cast<const volatile value_type &>(v_);
598
        platform_fence_after_load(order);
599
        return v;
600
    }
601

    
602
    value_type
603
    fetch_add(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
604
    {
605
        platform_fence_before(order);
606
        __asm__ __volatile__
607
        (
608
            "lock ; xaddl %0, %1"
609
            : "+r" (v), "+m" (v_)
610
            :
611
            : "cc"
612
        );
613
        platform_fence_after(order);
614
        return v;
615
    }
616

    
617
    value_type
618
    fetch_sub(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
619
    {
620
        return fetch_add(-v, order);
621
    }
622

    
623
    value_type
624
    exchange(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
625
    {
626
        platform_fence_before(order);
627
        __asm__ __volatile__
628
        (
629
            "xchgl %0, %1"
630
            : "+r" (v), "+m" (v_)
631
        );
632
        platform_fence_after(order);
633
        return v;
634
    }
635

    
636
    bool
637
    compare_exchange_strong(
638
        value_type & expected,
639
        value_type desired,
640
        memory_order success_order,
641
        memory_order failure_order) volatile BOOST_NOEXCEPT
642
    {
643
        value_type previous = expected;
644
        platform_fence_before(success_order);
645
        bool success;
646
        __asm__ __volatile__
647
        (
648
            "lock ; cmpxchgl %3, %1\n\t"
649
            "sete %2"
650
            : "+a,a" (previous), "+m,m" (v_), "=q,m" (success)
651
            : "r,r" (desired)
652
            : "cc"
653
        );
654
        if (success)
655
            platform_fence_after(success_order);
656
        else
657
            platform_fence_after(failure_order);
658
        expected = previous;
659
        return success;
660
    }
661

    
662
    bool
663
    compare_exchange_weak(
664
        value_type & expected,
665
        value_type desired,
666
        memory_order success_order,
667
        memory_order failure_order) volatile BOOST_NOEXCEPT
668
    {
669
        return compare_exchange_strong(expected, desired, success_order, failure_order);
670
    }
671

    
672
    value_type
673
    fetch_and(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
674
    {
675
        value_type tmp = load(memory_order_relaxed);
676
        while (!compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed))
677
        {
678
            BOOST_ATOMIC_X86_PAUSE();
679
        }
680
        return tmp;
681
    }
682

    
683
    value_type
684
    fetch_or(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
685
    {
686
        value_type tmp = load(memory_order_relaxed);
687
        while (!compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed))
688
        {
689
            BOOST_ATOMIC_X86_PAUSE();
690
        }
691
        return tmp;
692
    }
693

    
694
    value_type
695
    fetch_xor(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
696
    {
697
        value_type tmp = load(memory_order_relaxed);
698
        while (!compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed))
699
        {
700
            BOOST_ATOMIC_X86_PAUSE();
701
        }
702
        return tmp;
703
    }
704

    
705
    bool
706
    is_lock_free(void) const volatile BOOST_NOEXCEPT
707
    {
708
        return true;
709
    }
710

    
711
    BOOST_ATOMIC_DECLARE_INTEGRAL_OPERATORS
712

    
713
    BOOST_DELETED_FUNCTION(base_atomic(base_atomic const&))
714
    BOOST_DELETED_FUNCTION(base_atomic& operator=(base_atomic const&))
715

    
716
private:
717
    value_type v_;
718
};
719

    
720
#if defined(__x86_64__)
721
template<typename T, bool Sign>
722
class base_atomic<T, int, 8, Sign>
723
{
724
private:
725
    typedef base_atomic this_type;
726
    typedef T value_type;
727
    typedef T difference_type;
728

    
729
protected:
730
    typedef value_type value_arg_type;
731

    
732
public:
733
    BOOST_DEFAULTED_FUNCTION(base_atomic(void), {})
734
    BOOST_CONSTEXPR explicit base_atomic(value_type v) BOOST_NOEXCEPT : v_(v) {}
735

    
736
    void
737
    store(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
738
    {
739
        if (order != memory_order_seq_cst) {
740
            platform_fence_before(order);
741
            const_cast<volatile value_type &>(v_) = v;
742
        } else {
743
            exchange(v, order);
744
        }
745
    }
746

    
747
    value_type
748
    load(memory_order order = memory_order_seq_cst) const volatile BOOST_NOEXCEPT
749
    {
750
        value_type v = const_cast<const volatile value_type &>(v_);
751
        platform_fence_after_load(order);
752
        return v;
753
    }
754

    
755
    value_type
756
    fetch_add(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
757
    {
758
        platform_fence_before(order);
759
        __asm__ __volatile__
760
        (
761
            "lock ; xaddq %0, %1"
762
            : "+r" (v), "+m" (v_)
763
            :
764
            : "cc"
765
        );
766
        platform_fence_after(order);
767
        return v;
768
    }
769

    
770
    value_type
771
    fetch_sub(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
772
    {
773
        return fetch_add(-v, order);
774
    }
775

    
776
    value_type
777
    exchange(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
778
    {
779
        platform_fence_before(order);
780
        __asm__ __volatile__
781
        (
782
            "xchgq %0, %1"
783
            : "+r" (v), "+m" (v_)
784
        );
785
        platform_fence_after(order);
786
        return v;
787
    }
788

    
789
    bool
790
    compare_exchange_strong(
791
        value_type & expected,
792
        value_type desired,
793
        memory_order success_order,
794
        memory_order failure_order) volatile BOOST_NOEXCEPT
795
    {
796
        value_type previous = expected;
797
        platform_fence_before(success_order);
798
        bool success;
799
        __asm__ __volatile__
800
        (
801
            "lock ; cmpxchgq %3, %1\n\t"
802
            "sete %2"
803
            : "+a,a" (previous), "+m,m" (v_), "=q,m" (success)
804
            : "r,r" (desired)
805
            : "cc"
806
        );
807
        if (success)
808
            platform_fence_after(success_order);
809
        else
810
            platform_fence_after(failure_order);
811
        expected = previous;
812
        return success;
813
    }
814

    
815
    bool
816
    compare_exchange_weak(
817
        value_type & expected,
818
        value_type desired,
819
        memory_order success_order,
820
        memory_order failure_order) volatile BOOST_NOEXCEPT
821
    {
822
        return compare_exchange_strong(expected, desired, success_order, failure_order);
823
    }
824

    
825
    value_type
826
    fetch_and(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
827
    {
828
        value_type tmp = load(memory_order_relaxed);
829
        while (!compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed))
830
        {
831
            BOOST_ATOMIC_X86_PAUSE();
832
        }
833
        return tmp;
834
    }
835

    
836
    value_type
837
    fetch_or(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
838
    {
839
        value_type tmp = load(memory_order_relaxed);
840
        while (!compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed))
841
        {
842
            BOOST_ATOMIC_X86_PAUSE();
843
        }
844
        return tmp;
845
    }
846

    
847
    value_type
848
    fetch_xor(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
849
    {
850
        value_type tmp = load(memory_order_relaxed);
851
        while (!compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed))
852
        {
853
            BOOST_ATOMIC_X86_PAUSE();
854
        }
855
        return tmp;
856
    }
857

    
858
    bool
859
    is_lock_free(void) const volatile BOOST_NOEXCEPT
860
    {
861
        return true;
862
    }
863

    
864
    BOOST_ATOMIC_DECLARE_INTEGRAL_OPERATORS
865

    
866
    BOOST_DELETED_FUNCTION(base_atomic(base_atomic const&))
867
    BOOST_DELETED_FUNCTION(base_atomic& operator=(base_atomic const&))
868

    
869
private:
870
    value_type v_;
871
};
872

    
873
#endif
874

    
875
/* pointers */
876

    
877
// NOTE: x32 target is still regarded to as x86_64 and can only be detected by the size of pointers
878
#if !defined(__x86_64__) || (defined(__SIZEOF_POINTER__) && __SIZEOF_POINTER__ == 4)
879

    
880
template<bool Sign>
881
class base_atomic<void *, void *, 4, Sign>
882
{
883
private:
884
    typedef base_atomic this_type;
885
    typedef std::ptrdiff_t difference_type;
886
    typedef void * value_type;
887

    
888
protected:
889
    typedef value_type value_arg_type;
890

    
891
public:
892
    BOOST_DEFAULTED_FUNCTION(base_atomic(void), {})
893
    BOOST_CONSTEXPR explicit base_atomic(value_type v) BOOST_NOEXCEPT : v_(v) {}
894

    
895
    void
896
    store(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
897
    {
898
        if (order != memory_order_seq_cst) {
899
            platform_fence_before(order);
900
            const_cast<volatile value_type &>(v_) = v;
901
        } else {
902
            exchange(v, order);
903
        }
904
    }
905

    
906
    value_type load(memory_order order = memory_order_seq_cst) const volatile BOOST_NOEXCEPT
907
    {
908
        value_type v = const_cast<const volatile value_type &>(v_);
909
        platform_fence_after_load(order);
910
        return v;
911
    }
912

    
913
    value_type exchange(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
914
    {
915
        platform_fence_before(order);
916
        __asm__ __volatile__
917
        (
918
            "xchgl %0, %1"
919
            : "+r" (v), "+m" (v_)
920
        );
921
        platform_fence_after(order);
922
        return v;
923
    }
924

    
925
    bool compare_exchange_strong(value_type & expected, value_type desired,
926
        memory_order success_order,
927
        memory_order failure_order) volatile BOOST_NOEXCEPT
928
    {
929
        value_type previous = expected;
930
        platform_fence_before(success_order);
931
        bool success;
932
        __asm__ __volatile__
933
        (
934
            "lock ; cmpxchgl %3, %1\n\t"
935
            "sete %2"
936
            : "+a,a" (previous), "+m,m" (v_), "=q,m" (success)
937
            : "r,r" (desired)
938
            : "cc"
939
        );
940
        if (success)
941
            platform_fence_after(success_order);
942
        else
943
            platform_fence_after(failure_order);
944
        expected = previous;
945
        return success;
946
    }
947

    
948
    bool compare_exchange_weak(value_type & expected, value_type desired,
949
        memory_order success_order,
950
        memory_order failure_order) volatile BOOST_NOEXCEPT
951
    {
952
        return compare_exchange_strong(expected, desired, success_order, failure_order);
953
    }
954

    
955
    bool
956
    is_lock_free(void) const volatile BOOST_NOEXCEPT
957
    {
958
        return true;
959
    }
960

    
961
    value_type
962
    fetch_add(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
963
    {
964
        platform_fence_before(order);
965
        __asm__ __volatile__
966
        (
967
            "lock ; xaddl %0, %1"
968
            : "+r" (v), "+m" (v_)
969
            :
970
            : "cc"
971
        );
972
        platform_fence_after(order);
973
        return reinterpret_cast<value_type>(v);
974
    }
975

    
976
    value_type
977
    fetch_sub(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
978
    {
979
        return fetch_add(-v, order);
980
    }
981

    
982
    BOOST_ATOMIC_DECLARE_VOID_POINTER_OPERATORS
983

    
984
    BOOST_DELETED_FUNCTION(base_atomic(base_atomic const&))
985
    BOOST_DELETED_FUNCTION(base_atomic& operator=(base_atomic const&))
986

    
987
private:
988
    value_type v_;
989
};
990

    
991
template<typename T, bool Sign>
992
class base_atomic<T *, void *, 4, Sign>
993
{
994
private:
995
    typedef base_atomic this_type;
996
    typedef T * value_type;
997
    typedef std::ptrdiff_t difference_type;
998

    
999
protected:
1000
    typedef value_type value_arg_type;
1001

    
1002
public:
1003
    BOOST_DEFAULTED_FUNCTION(base_atomic(void), {})
1004
    BOOST_CONSTEXPR explicit base_atomic(value_type v) BOOST_NOEXCEPT : v_(v) {}
1005

    
1006
    void
1007
    store(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
1008
    {
1009
        if (order != memory_order_seq_cst) {
1010
            platform_fence_before(order);
1011
            const_cast<volatile value_type &>(v_) = v;
1012
        } else {
1013
            exchange(v, order);
1014
        }
1015
    }
1016

    
1017
    value_type
1018
    load(memory_order order = memory_order_seq_cst) const volatile BOOST_NOEXCEPT
1019
    {
1020
        value_type v = const_cast<const volatile value_type &>(v_);
1021
        platform_fence_after_load(order);
1022
        return v;
1023
    }
1024

    
1025
    value_type
1026
    exchange(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
1027
    {
1028
        platform_fence_before(order);
1029
        __asm__ __volatile__
1030
        (
1031
            "xchgl %0, %1"
1032
            : "+r" (v), "+m" (v_)
1033
        );
1034
        platform_fence_after(order);
1035
        return v;
1036
    }
1037

    
1038
    bool
1039
    compare_exchange_strong(
1040
        value_type & expected,
1041
        value_type desired,
1042
        memory_order success_order,
1043
        memory_order failure_order) volatile BOOST_NOEXCEPT
1044
    {
1045
        value_type previous = expected;
1046
        platform_fence_before(success_order);
1047
        bool success;
1048
        __asm__ __volatile__
1049
        (
1050
            "lock ; cmpxchgl %3, %1\n\t"
1051
            "sete %2"
1052
            : "+a,a" (previous), "+m,m" (v_), "=q,m" (success)
1053
            : "r,r" (desired)
1054
            : "cc"
1055
        );
1056
        if (success)
1057
            platform_fence_after(success_order);
1058
        else
1059
            platform_fence_after(failure_order);
1060
        expected = previous;
1061
        return success;
1062
    }
1063

    
1064
    bool
1065
    compare_exchange_weak(
1066
        value_type & expected,
1067
        value_type desired,
1068
        memory_order success_order,
1069
        memory_order failure_order) volatile BOOST_NOEXCEPT
1070
    {
1071
        return compare_exchange_strong(expected, desired, success_order, failure_order);
1072
    }
1073

    
1074
    value_type
1075
    fetch_add(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
1076
    {
1077
        v = v * sizeof(*v_);
1078
        platform_fence_before(order);
1079
        __asm__ __volatile__
1080
        (
1081
            "lock ; xaddl %0, %1"
1082
            : "+r" (v), "+m" (v_)
1083
            :
1084
            : "cc"
1085
        );
1086
        platform_fence_after(order);
1087
        return reinterpret_cast<value_type>(v);
1088
    }
1089

    
1090
    value_type
1091
    fetch_sub(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
1092
    {
1093
        return fetch_add(-v, order);
1094
    }
1095

    
1096
    bool
1097
    is_lock_free(void) const volatile BOOST_NOEXCEPT
1098
    {
1099
        return true;
1100
    }
1101

    
1102
    BOOST_ATOMIC_DECLARE_POINTER_OPERATORS
1103

    
1104
    BOOST_DELETED_FUNCTION(base_atomic(base_atomic const&))
1105
    BOOST_DELETED_FUNCTION(base_atomic& operator=(base_atomic const&))
1106

    
1107
private:
1108
    value_type v_;
1109
};
1110

    
1111
#else
1112

    
1113
template<bool Sign>
1114
class base_atomic<void *, void *, 8, Sign>
1115
{
1116
private:
1117
    typedef base_atomic this_type;
1118
    typedef std::ptrdiff_t difference_type;
1119
    typedef void * value_type;
1120

    
1121
protected:
1122
    typedef value_type value_arg_type;
1123

    
1124
public:
1125
    BOOST_DEFAULTED_FUNCTION(base_atomic(void), {})
1126
    BOOST_CONSTEXPR explicit base_atomic(value_type v) BOOST_NOEXCEPT : v_(v) {}
1127

    
1128
    void
1129
    store(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
1130
    {
1131
        if (order != memory_order_seq_cst) {
1132
            platform_fence_before(order);
1133
            const_cast<volatile value_type &>(v_) = v;
1134
        } else {
1135
            exchange(v, order);
1136
        }
1137
    }
1138

    
1139
    value_type load(memory_order order = memory_order_seq_cst) const volatile BOOST_NOEXCEPT
1140
    {
1141
        value_type v = const_cast<const volatile value_type &>(v_);
1142
        platform_fence_after_load(order);
1143
        return v;
1144
    }
1145

    
1146
    value_type exchange(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
1147
    {
1148
        platform_fence_before(order);
1149
        __asm__ __volatile__
1150
        (
1151
            "xchgq %0, %1"
1152
            : "+r" (v), "+m" (v_)
1153
        );
1154
        platform_fence_after(order);
1155
        return v;
1156
    }
1157

    
1158
    bool compare_exchange_strong(value_type & expected, value_type desired,
1159
        memory_order success_order,
1160
        memory_order failure_order) volatile BOOST_NOEXCEPT
1161
    {
1162
        value_type previous = expected;
1163
        platform_fence_before(success_order);
1164
        bool success;
1165
        __asm__ __volatile__
1166
        (
1167
            "lock ; cmpxchgq %3, %1\n\t"
1168
            "sete %2"
1169
            : "+a,a" (previous), "+m,m" (v_), "=q,m" (success)
1170
            : "r,r" (desired)
1171
            : "cc"
1172
        );
1173
        if (success)
1174
            platform_fence_after(success_order);
1175
        else
1176
            platform_fence_after(failure_order);
1177
        expected = previous;
1178
        return success;
1179
    }
1180

    
1181
    bool compare_exchange_weak(value_type & expected, value_type desired,
1182
        memory_order success_order,
1183
        memory_order failure_order) volatile BOOST_NOEXCEPT
1184
    {
1185
        return compare_exchange_strong(expected, desired, success_order, failure_order);
1186
    }
1187

    
1188
    bool
1189
    is_lock_free(void) const volatile BOOST_NOEXCEPT
1190
    {
1191
        return true;
1192
    }
1193

    
1194
    value_type
1195
    fetch_add(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
1196
    {
1197
        platform_fence_before(order);
1198
        __asm__ __volatile__
1199
        (
1200
            "lock ; xaddq %0, %1"
1201
            : "+r" (v), "+m" (v_)
1202
            :
1203
            : "cc"
1204
        );
1205
        platform_fence_after(order);
1206
        return reinterpret_cast<value_type>(v);
1207
    }
1208

    
1209
    value_type
1210
    fetch_sub(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
1211
    {
1212
        return fetch_add(-v, order);
1213
    }
1214

    
1215
    BOOST_ATOMIC_DECLARE_VOID_POINTER_OPERATORS
1216

    
1217
    BOOST_DELETED_FUNCTION(base_atomic(base_atomic const&))
1218
    BOOST_DELETED_FUNCTION(base_atomic& operator=(base_atomic const&))
1219

    
1220
private:
1221
    value_type v_;
1222
};
1223

    
1224
template<typename T, bool Sign>
1225
class base_atomic<T *, void *, 8, Sign>
1226
{
1227
private:
1228
    typedef base_atomic this_type;
1229
    typedef T * value_type;
1230
    typedef std::ptrdiff_t difference_type;
1231

    
1232
protected:
1233
    typedef value_type value_arg_type;
1234

    
1235
public:
1236
    BOOST_DEFAULTED_FUNCTION(base_atomic(void), {})
1237
    BOOST_CONSTEXPR explicit base_atomic(value_type v) BOOST_NOEXCEPT : v_(v) {}
1238

    
1239
    void
1240
    store(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
1241
    {
1242
        if (order != memory_order_seq_cst) {
1243
            platform_fence_before(order);
1244
            const_cast<volatile value_type &>(v_) = v;
1245
        } else {
1246
            exchange(v, order);
1247
        }
1248
    }
1249

    
1250
    value_type
1251
    load(memory_order order = memory_order_seq_cst) const volatile BOOST_NOEXCEPT
1252
    {
1253
        value_type v = const_cast<const volatile value_type &>(v_);
1254
        platform_fence_after_load(order);
1255
        return v;
1256
    }
1257

    
1258
    value_type
1259
    exchange(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
1260
    {
1261
        platform_fence_before(order);
1262
        __asm__ __volatile__
1263
        (
1264
            "xchgq %0, %1"
1265
            : "+r" (v), "+m" (v_)
1266
        );
1267
        platform_fence_after(order);
1268
        return v;
1269
    }
1270

    
1271
    bool
1272
    compare_exchange_strong(
1273
        value_type & expected,
1274
        value_type desired,
1275
        memory_order success_order,
1276
        memory_order failure_order) volatile BOOST_NOEXCEPT
1277
    {
1278
        value_type previous = expected;
1279
        platform_fence_before(success_order);
1280
        bool success;
1281
        __asm__ __volatile__
1282
        (
1283
            "lock ; cmpxchgq %3, %1\n\t"
1284
            "sete %2"
1285
            : "+a,a" (previous), "+m,m" (v_), "=q,m" (success)
1286
            : "r,r" (desired)
1287
            : "cc"
1288
        );
1289
        if (success)
1290
            platform_fence_after(success_order);
1291
        else
1292
            platform_fence_after(failure_order);
1293
        expected = previous;
1294
        return success;
1295
    }
1296

    
1297
    bool
1298
    compare_exchange_weak(
1299
        value_type & expected,
1300
        value_type desired,
1301
        memory_order success_order,
1302
        memory_order failure_order) volatile BOOST_NOEXCEPT
1303
    {
1304
        return compare_exchange_strong(expected, desired, success_order, failure_order);
1305
    }
1306

    
1307
    value_type
1308
    fetch_add(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
1309
    {
1310
        v = v * sizeof(*v_);
1311
        platform_fence_before(order);
1312
        __asm__ __volatile__
1313
        (
1314
            "lock ; xaddq %0, %1"
1315
            : "+r" (v), "+m" (v_)
1316
            :
1317
            : "cc"
1318
        );
1319
        platform_fence_after(order);
1320
        return reinterpret_cast<value_type>(v);
1321
    }
1322

    
1323
    value_type
1324
    fetch_sub(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
1325
    {
1326
        return fetch_add(-v, order);
1327
    }
1328

    
1329
    bool
1330
    is_lock_free(void) const volatile BOOST_NOEXCEPT
1331
    {
1332
        return true;
1333
    }
1334

    
1335
    BOOST_ATOMIC_DECLARE_POINTER_OPERATORS
1336

    
1337
    BOOST_DELETED_FUNCTION(base_atomic(base_atomic const&))
1338
    BOOST_DELETED_FUNCTION(base_atomic& operator=(base_atomic const&))
1339

    
1340
private:
1341
    value_type v_;
1342
};
1343

    
1344
#endif
1345

    
1346
template<typename T, bool Sign>
1347
class base_atomic<T, void, 1, Sign>
1348
{
1349
private:
1350
    typedef base_atomic this_type;
1351
    typedef T value_type;
1352
    typedef uint8_t storage_type;
1353

    
1354
protected:
1355
    typedef value_type const& value_arg_type;
1356

    
1357
public:
1358
    BOOST_DEFAULTED_FUNCTION(base_atomic(void), {})
1359
    BOOST_CONSTEXPR explicit base_atomic(value_type const& v) BOOST_NOEXCEPT :
1360
        v_(reinterpret_cast<storage_type const&>(v))
1361
    {
1362
    }
1363

    
1364
    void
1365
    store(value_type const& v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
1366
    {
1367
        if (order != memory_order_seq_cst) {
1368
            storage_type tmp;
1369
            memcpy(&tmp, &v, sizeof(value_type));
1370
            platform_fence_before(order);
1371
            const_cast<volatile storage_type &>(v_) = tmp;
1372
        } else {
1373
            exchange(v, order);
1374
        }
1375
    }
1376

    
1377
    value_type
1378
    load(memory_order order = memory_order_seq_cst) const volatile BOOST_NOEXCEPT
1379
    {
1380
        storage_type tmp = const_cast<volatile storage_type &>(v_);
1381
        platform_fence_after_load(order);
1382
        value_type v;
1383
        memcpy(&v, &tmp, sizeof(value_type));
1384
        return v;
1385
    }
1386

    
1387
    value_type
1388
    exchange(value_type const& v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
1389
    {
1390
        storage_type tmp;
1391
        memcpy(&tmp, &v, sizeof(value_type));
1392
        platform_fence_before(order);
1393
        __asm__ __volatile__
1394
        (
1395
            "xchgb %0, %1"
1396
            : "+q" (tmp), "+m" (v_)
1397
        );
1398
        platform_fence_after(order);
1399
        value_type res;
1400
        memcpy(&res, &tmp, sizeof(value_type));
1401
        return res;
1402
    }
1403

    
1404
    bool
1405
    compare_exchange_strong(
1406
        value_type & expected,
1407
        value_type const& desired,
1408
        memory_order success_order,
1409
        memory_order failure_order) volatile BOOST_NOEXCEPT
1410
    {
1411
        storage_type expected_s, desired_s;
1412
        memcpy(&expected_s, &expected, sizeof(value_type));
1413
        memcpy(&desired_s, &desired, sizeof(value_type));
1414
        storage_type previous_s = expected_s;
1415
        platform_fence_before(success_order);
1416
        bool success;
1417
        __asm__ __volatile__
1418
        (
1419
            "lock ; cmpxchgb %3, %1\n\t"
1420
            "sete %2"
1421
            : "+a" (previous_s), "+m" (v_), "=q" (success)
1422
            : "q" (desired_s)
1423
            : "cc"
1424
        );
1425
        if (success)
1426
            platform_fence_after(success_order);
1427
        else
1428
            platform_fence_after(failure_order);
1429
        memcpy(&expected, &previous_s, sizeof(value_type));
1430
        return success;
1431
    }
1432

    
1433
    bool
1434
    compare_exchange_weak(
1435
        value_type & expected,
1436
        value_type const& desired,
1437
        memory_order success_order,
1438
        memory_order failure_order) volatile BOOST_NOEXCEPT
1439
    {
1440
        return compare_exchange_strong(expected, desired, success_order, failure_order);
1441
    }
1442

    
1443
    bool
1444
    is_lock_free(void) const volatile BOOST_NOEXCEPT
1445
    {
1446
        return true;
1447
    }
1448

    
1449
    BOOST_ATOMIC_DECLARE_BASE_OPERATORS
1450

    
1451
    BOOST_DELETED_FUNCTION(base_atomic(base_atomic const&))
1452
    BOOST_DELETED_FUNCTION(base_atomic& operator=(base_atomic const&))
1453

    
1454
private:
1455
    storage_type v_;
1456
};
1457

    
1458
template<typename T, bool Sign>
1459
class base_atomic<T, void, 2, Sign>
1460
{
1461
private:
1462
    typedef base_atomic this_type;
1463
    typedef T value_type;
1464
    typedef uint16_t storage_type;
1465

    
1466
protected:
1467
    typedef value_type const& value_arg_type;
1468

    
1469
public:
1470
    BOOST_DEFAULTED_FUNCTION(base_atomic(void), {})
1471
    BOOST_CONSTEXPR explicit base_atomic(value_type const& v) BOOST_NOEXCEPT :
1472
        v_(reinterpret_cast<storage_type const&>(v))
1473
    {
1474
    }
1475

    
1476
    void
1477
    store(value_type const& v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
1478
    {
1479
        if (order != memory_order_seq_cst) {
1480
            storage_type tmp;
1481
            memcpy(&tmp, &v, sizeof(value_type));
1482
            platform_fence_before(order);
1483
            const_cast<volatile storage_type &>(v_) = tmp;
1484
        } else {
1485
            exchange(v, order);
1486
        }
1487
    }
1488

    
1489
    value_type
1490
    load(memory_order order = memory_order_seq_cst) const volatile BOOST_NOEXCEPT
1491
    {
1492
        storage_type tmp = const_cast<volatile storage_type &>(v_);
1493
        platform_fence_after_load(order);
1494
        value_type v;
1495
        memcpy(&v, &tmp, sizeof(value_type));
1496
        return v;
1497
    }
1498

    
1499
    value_type
1500
    exchange(value_type const& v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
1501
    {
1502
        storage_type tmp;
1503
        memcpy(&tmp, &v, sizeof(value_type));
1504
        platform_fence_before(order);
1505
        __asm__ __volatile__
1506
        (
1507
            "xchgw %0, %1"
1508
            : "+q" (tmp), "+m" (v_)
1509
        );
1510
        platform_fence_after(order);
1511
        value_type res;
1512
        memcpy(&res, &tmp, sizeof(value_type));
1513
        return res;
1514
    }
1515

    
1516
    bool
1517
    compare_exchange_strong(
1518
        value_type & expected,
1519
        value_type const& desired,
1520
        memory_order success_order,
1521
        memory_order failure_order) volatile BOOST_NOEXCEPT
1522
    {
1523
        storage_type expected_s, desired_s;
1524
        memcpy(&expected_s, &expected, sizeof(value_type));
1525
        memcpy(&desired_s, &desired, sizeof(value_type));
1526
        storage_type previous_s = expected_s;
1527
        platform_fence_before(success_order);
1528
        bool success;
1529
        __asm__ __volatile__
1530
        (
1531
            "lock ; cmpxchgw %3, %1\n\t"
1532
            "sete %2"
1533
            : "+a" (previous_s), "+m" (v_), "=q" (success)
1534
            : "q" (desired_s)
1535
            : "cc"
1536
        );
1537
        if (success)
1538
            platform_fence_after(success_order);
1539
        else
1540
            platform_fence_after(failure_order);
1541
        memcpy(&expected, &previous_s, sizeof(value_type));
1542
        return success;
1543
    }
1544

    
1545
    bool
1546
    compare_exchange_weak(
1547
        value_type & expected,
1548
        value_type const& desired,
1549
        memory_order success_order,
1550
        memory_order failure_order) volatile BOOST_NOEXCEPT
1551
    {
1552
        return compare_exchange_strong(expected, desired, success_order, failure_order);
1553
    }
1554

    
1555
    bool
1556
    is_lock_free(void) const volatile BOOST_NOEXCEPT
1557
    {
1558
        return true;
1559
    }
1560

    
1561
    BOOST_ATOMIC_DECLARE_BASE_OPERATORS
1562

    
1563
    BOOST_DELETED_FUNCTION(base_atomic(base_atomic const&))
1564
    BOOST_DELETED_FUNCTION(base_atomic& operator=(base_atomic const&))
1565

    
1566
private:
1567
    storage_type v_;
1568
};
1569

    
1570
template<typename T, bool Sign>
1571
class base_atomic<T, void, 4, Sign>
1572
{
1573
private:
1574
    typedef base_atomic this_type;
1575
    typedef T value_type;
1576
    typedef uint32_t storage_type;
1577

    
1578
protected:
1579
    typedef value_type const& value_arg_type;
1580

    
1581
public:
1582
    BOOST_DEFAULTED_FUNCTION(base_atomic(void), {})
1583
    explicit base_atomic(value_type const& v) BOOST_NOEXCEPT : v_(0)
1584
    {
1585
        memcpy(&v_, &v, sizeof(value_type));
1586
    }
1587

    
1588
    void
1589
    store(value_type const& v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
1590
    {
1591
        if (order != memory_order_seq_cst) {
1592
            storage_type tmp = 0;
1593
            memcpy(&tmp, &v, sizeof(value_type));
1594
            platform_fence_before(order);
1595
            const_cast<volatile storage_type &>(v_) = tmp;
1596
        } else {
1597
            exchange(v, order);
1598
        }
1599
    }
1600

    
1601
    value_type
1602
    load(memory_order order = memory_order_seq_cst) const volatile BOOST_NOEXCEPT
1603
    {
1604
        storage_type tmp = const_cast<volatile storage_type &>(v_);
1605
        platform_fence_after_load(order);
1606
        value_type v;
1607
        memcpy(&v, &tmp, sizeof(value_type));
1608
        return v;
1609
    }
1610

    
1611
    value_type
1612
    exchange(value_type const& v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
1613
    {
1614
        storage_type tmp = 0;
1615
        memcpy(&tmp, &v, sizeof(value_type));
1616
        platform_fence_before(order);
1617
        __asm__ __volatile__
1618
        (
1619
            "xchgl %0, %1"
1620
            : "+q" (tmp), "+m" (v_)
1621
        );
1622
        platform_fence_after(order);
1623
        value_type res;
1624
        memcpy(&res, &tmp, sizeof(value_type));
1625
        return res;
1626
    }
1627

    
1628
    bool
1629
    compare_exchange_strong(
1630
        value_type & expected,
1631
        value_type const& desired,
1632
        memory_order success_order,
1633
        memory_order failure_order) volatile BOOST_NOEXCEPT
1634
    {
1635
        storage_type expected_s = 0, desired_s = 0;
1636
        memcpy(&expected_s, &expected, sizeof(value_type));
1637
        memcpy(&desired_s, &desired, sizeof(value_type));
1638
        storage_type previous_s = expected_s;
1639
        platform_fence_before(success_order);
1640
        bool success;
1641
        __asm__ __volatile__
1642
        (
1643
            "lock ; cmpxchgl %3, %1\n\t"
1644
            "sete %2"
1645
            : "+a,a" (previous_s), "+m,m" (v_), "=q,m" (success)
1646
            : "q,q" (desired_s)
1647
            : "cc"
1648
        );
1649
        if (success)
1650
            platform_fence_after(success_order);
1651
        else
1652
            platform_fence_after(failure_order);
1653
        memcpy(&expected, &previous_s, sizeof(value_type));
1654
        return success;
1655
    }
1656

    
1657
    bool
1658
    compare_exchange_weak(
1659
        value_type & expected,
1660
        value_type const& desired,
1661
        memory_order success_order,
1662
        memory_order failure_order) volatile BOOST_NOEXCEPT
1663
    {
1664
        return compare_exchange_strong(expected, desired, success_order, failure_order);
1665
    }
1666

    
1667
    bool
1668
    is_lock_free(void) const volatile BOOST_NOEXCEPT
1669
    {
1670
        return true;
1671
    }
1672

    
1673
    BOOST_ATOMIC_DECLARE_BASE_OPERATORS
1674

    
1675
    BOOST_DELETED_FUNCTION(base_atomic(base_atomic const&))
1676
    BOOST_DELETED_FUNCTION(base_atomic& operator=(base_atomic const&))
1677

    
1678
private:
1679
    storage_type v_;
1680
};
1681

    
1682
#if defined(__x86_64__)
1683
template<typename T, bool Sign>
1684
class base_atomic<T, void, 8, Sign>
1685
{
1686
private:
1687
    typedef base_atomic this_type;
1688
    typedef T value_type;
1689
    typedef uint64_t storage_type;
1690

    
1691
protected:
1692
    typedef value_type const& value_arg_type;
1693

    
1694
public:
1695
    BOOST_DEFAULTED_FUNCTION(base_atomic(void), {})
1696
    explicit base_atomic(value_type const& v) BOOST_NOEXCEPT : v_(0)
1697
    {
1698
        memcpy(&v_, &v, sizeof(value_type));
1699
    }
1700

    
1701
    void
1702
    store(value_type const& v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
1703
    {
1704
        if (order != memory_order_seq_cst) {
1705
            storage_type tmp = 0;
1706
            memcpy(&tmp, &v, sizeof(value_type));
1707
            platform_fence_before(order);
1708
            const_cast<volatile storage_type &>(v_) = tmp;
1709
        } else {
1710
            exchange(v, order);
1711
        }
1712
    }
1713

    
1714
    value_type
1715
    load(memory_order order = memory_order_seq_cst) const volatile BOOST_NOEXCEPT
1716
    {
1717
        storage_type tmp = const_cast<volatile storage_type &>(v_);
1718
        platform_fence_after_load(order);
1719
        value_type v;
1720
        memcpy(&v, &tmp, sizeof(value_type));
1721
        return v;
1722
    }
1723

    
1724
    value_type
1725
    exchange(value_type const& v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
1726
    {
1727
        storage_type tmp = 0;
1728
        memcpy(&tmp, &v, sizeof(value_type));
1729
        platform_fence_before(order);
1730
        __asm__ __volatile__
1731
        (
1732
            "xchgq %0, %1"
1733
            : "+q" (tmp), "+m" (v_)
1734
        );
1735
        platform_fence_after(order);
1736
        value_type res;
1737
        memcpy(&res, &tmp, sizeof(value_type));
1738
        return res;
1739
    }
1740

    
1741
    bool
1742
    compare_exchange_strong(
1743
        value_type & expected,
1744
        value_type const& desired,
1745
        memory_order success_order,
1746
        memory_order failure_order) volatile BOOST_NOEXCEPT
1747
    {
1748
        storage_type expected_s = 0, desired_s = 0;
1749
        memcpy(&expected_s, &expected, sizeof(value_type));
1750
        memcpy(&desired_s, &desired, sizeof(value_type));
1751
        storage_type previous_s = expected_s;
1752
        platform_fence_before(success_order);
1753
        bool success;
1754
        __asm__ __volatile__
1755
        (
1756
            "lock ; cmpxchgq %3, %1\n\t"
1757
            "sete %2"
1758
            : "+a,a" (previous_s), "+m,m" (v_), "=q,m" (success)
1759
            : "q,q" (desired_s)
1760
            : "cc"
1761
        );
1762
        if (success)
1763
            platform_fence_after(success_order);
1764
        else
1765
            platform_fence_after(failure_order);
1766
        memcpy(&expected, &previous_s, sizeof(value_type));
1767
        return success;
1768
    }
1769

    
1770
    bool
1771
    compare_exchange_weak(
1772
        value_type & expected,
1773
        value_type const& desired,
1774
        memory_order success_order,
1775
        memory_order failure_order) volatile BOOST_NOEXCEPT
1776
    {
1777
        return compare_exchange_strong(expected, desired, success_order, failure_order);
1778
    }
1779

    
1780
    bool
1781
    is_lock_free(void) const volatile BOOST_NOEXCEPT
1782
    {
1783
        return true;
1784
    }
1785

    
1786
    BOOST_ATOMIC_DECLARE_BASE_OPERATORS
1787

    
1788
    BOOST_DELETED_FUNCTION(base_atomic(base_atomic const&))
1789
    BOOST_DELETED_FUNCTION(base_atomic& operator=(base_atomic const&))
1790

    
1791
private:
1792
    storage_type v_;
1793
};
1794
#endif
1795

    
1796
#if !defined(__x86_64__) && defined(BOOST_ATOMIC_X86_HAS_CMPXCHG8B)
1797

    
1798
template<typename T>
1799
inline bool
1800
platform_cmpxchg64_strong(T & expected, T desired, volatile T * ptr) BOOST_NOEXCEPT
1801
{
1802
#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8
1803
    const T oldval = __sync_val_compare_and_swap(ptr, expected, desired);
1804
    const bool result = (oldval == expected);
1805
    expected = oldval;
1806
    return result;
1807
#else
1808
    uint32_t scratch;
1809
    /* Make sure ebx is saved and restored properly in case
1810
    this object is compiled as "position independent". Since
1811
    programmers on x86 tend to forget specifying -DPIC or
1812
    similar, always assume PIC.
1813

1814
    To make this work uniformly even in the non-PIC case,
1815
    setup register constraints such that ebx can not be
1816
    used by accident e.g. as base address for the variable
1817
    to be modified. Accessing "scratch" should always be okay,
1818
    as it can only be placed on the stack (and therefore
1819
    accessed through ebp or esp only).
1820

1821
    In theory, could push/pop ebx onto/off the stack, but movs
1822
    to a prepared stack slot turn out to be faster. */
1823
    bool success;
1824
    __asm__ __volatile__
1825
    (
1826
        "movl %%ebx, %[scratch]\n\t"
1827
        "movl %[desired_lo], %%ebx\n\t"
1828
        "lock; cmpxchg8b %[dest]\n\t"
1829
        "movl %[scratch], %%ebx\n\t"
1830
        "sete %[success]"
1831
        : "+A,A,A,A,A,A" (expected), [dest] "+m,m,m,m,m,m" (*ptr), [scratch] "=m,m,m,m,m,m" (scratch), [success] "=q,m,q,m,q,m" (success)
1832
        : [desired_lo] "S,S,D,D,m,m" ((uint32_t)desired), "c,c,c,c,c,c" ((uint32_t)(desired >> 32))
1833
        : "memory", "cc"
1834
    );
1835
    return success;
1836
#endif
1837
}
1838

    
1839
// Intel 64 and IA-32 Architectures Software Developer's Manual, Volume 3A, 8.1.1. Guaranteed Atomic Operations:
1840
//
1841
// The Pentium processor (and newer processors since) guarantees that the following additional memory operations will always be carried out atomically:
1842
// * Reading or writing a quadword aligned on a 64-bit boundary
1843
//
1844
// Luckily, the memory is almost always 8-byte aligned in our case because atomic<> uses 64 bit native types for storage and dynamic memory allocations
1845
// have at least 8 byte alignment. The only unfortunate case is when atomic is placeod on the stack and it is not 8-byte aligned (like on 32 bit Windows).
1846

    
1847
template<typename T>
1848
inline void
1849
platform_store64(T value, volatile T * ptr) BOOST_NOEXCEPT
1850
{
1851
    if (((uint32_t)ptr & 0x00000007) == 0)
1852
    {
1853
#if defined(__SSE2__)
1854
        __asm__ __volatile__
1855
        (
1856
            "movq %1, %%xmm4\n\t"
1857
            "movq %%xmm4, %0\n\t"
1858
            : "=m" (*ptr)
1859
            : "m" (value)
1860
            : "memory", "xmm4"
1861
        );
1862
#else
1863
        __asm__ __volatile__
1864
        (
1865
            "fildll %1\n\t"
1866
            "fistpll %0\n\t"
1867
            : "=m" (*ptr)
1868
            : "m" (value)
1869
            : "memory"
1870
        );
1871
#endif
1872
    }
1873
    else
1874
    {
1875
        uint32_t scratch;
1876
        __asm__ __volatile__
1877
        (
1878
            "movl %%ebx, %[scratch]\n\t"
1879
            "movl %[value_lo], %%ebx\n\t"
1880
            "movl 0(%[dest]), %%eax\n\t"
1881
            "movl 4(%[dest]), %%edx\n\t"
1882
            ".align 16\n\t"
1883
            "1: lock; cmpxchg8b 0(%[dest])\n\t"
1884
            "jne 1b\n\t"
1885
            "movl %[scratch], %%ebx"
1886
            : [scratch] "=m,m" (scratch)
1887
            : [value_lo] "a,a" ((uint32_t)value), "c,c" ((uint32_t)(value >> 32)), [dest] "D,S" (ptr)
1888
            : "memory", "cc", "edx"
1889
        );
1890
    }
1891
}
1892

    
1893
template<typename T>
1894
inline T
1895
platform_load64(const volatile T * ptr) BOOST_NOEXCEPT
1896
{
1897
    T value;
1898

    
1899
    if (((uint32_t)ptr & 0x00000007) == 0)
1900
    {
1901
#if defined(__SSE2__)
1902
        __asm__ __volatile__
1903
        (
1904
            "movq %1, %%xmm4\n\t"
1905
            "movq %%xmm4, %0\n\t"
1906
            : "=m" (value)
1907
            : "m" (*ptr)
1908
            : "memory", "xmm4"
1909
        );
1910
#else
1911
        __asm__ __volatile__
1912
        (
1913
            "fildll %1\n\t"
1914
            "fistpll %0\n\t"
1915
            : "=m" (value)
1916
            : "m" (*ptr)
1917
            : "memory"
1918
        );
1919
#endif
1920
    }
1921
    else
1922
    {
1923
        // We don't care for comparison result here; the previous value will be stored into value anyway.
1924
        // Also we don't care for ebx and ecx values, they just have to be equal to eax and edx before cmpxchg8b.
1925
        __asm__ __volatile__
1926
        (
1927
            "movl %%ebx, %%eax\n\t"
1928
            "movl %%ecx, %%edx\n\t"
1929
            "lock; cmpxchg8b %[dest]"
1930
            : "=&A" (value)
1931
            : [dest] "m" (*ptr)
1932
            : "cc"
1933
        );
1934
    }
1935

    
1936
    return value;
1937
}
1938

    
1939
#endif
1940

    
1941
#if defined(BOOST_ATOMIC_INT128_LOCK_FREE) && BOOST_ATOMIC_INT128_LOCK_FREE > 0
1942

    
1943
template<typename T>
1944
inline bool
1945
platform_cmpxchg128_strong(T& expected, T desired, volatile T* ptr) BOOST_NOEXCEPT
1946
{
1947
    uint64_t const* p_desired = (uint64_t const*)&desired;
1948
    bool success;
1949
    __asm__ __volatile__
1950
    (
1951
        "lock; cmpxchg16b %[dest]\n\t"
1952
        "sete %[success]"
1953
        : "+A,A" (expected), [dest] "+m,m" (*ptr), [success] "=q,m" (success)
1954
        : "b,b" (p_desired[0]), "c,c" (p_desired[1])
1955
        : "memory", "cc"
1956
    );
1957
    return success;
1958
}
1959

    
1960
template<typename T>
1961
inline void
1962
platform_store128(T value, volatile T* ptr) BOOST_NOEXCEPT
1963
{
1964
    uint64_t const* p_value = (uint64_t const*)&value;
1965
    __asm__ __volatile__
1966
    (
1967
        "movq 0(%[dest]), %%rax\n\t"
1968
        "movq 8(%[dest]), %%rdx\n\t"
1969
        ".align 16\n\t"
1970
        "1: lock; cmpxchg16b 0(%[dest])\n\t"
1971
        "jne 1b"
1972
        :
1973
        : "b" (p_value[0]), "c" (p_value[1]), [dest] "r" (ptr)
1974
        : "memory", "cc", "rax", "rdx"
1975
    );
1976
}
1977

    
1978
template<typename T>
1979
inline T
1980
platform_load128(const volatile T* ptr) BOOST_NOEXCEPT
1981
{
1982
    T value;
1983

    
1984
    // We don't care for comparison result here; the previous value will be stored into value anyway.
1985
    // Also we don't care for rbx and rcx values, they just have to be equal to rax and rdx before cmpxchg16b.
1986
    __asm__ __volatile__
1987
    (
1988
        "movq %%rbx, %%rax\n\t"
1989
        "movq %%rcx, %%rdx\n\t"
1990
        "lock; cmpxchg16b %[dest]"
1991
        : "=&A" (value)
1992
        : [dest] "m" (*ptr)
1993
        : "cc"
1994
    );
1995

    
1996
    return value;
1997
}
1998

    
1999
#endif // defined(BOOST_ATOMIC_INT128_LOCK_FREE) && BOOST_ATOMIC_INT128_LOCK_FREE > 0
2000

    
2001
}
2002
}
2003
}
2004

    
2005
/* pull in 64-bit atomic type using cmpxchg8b above */
2006
#if !defined(__x86_64__) && defined(BOOST_ATOMIC_X86_HAS_CMPXCHG8B)
2007
#include <boost/atomic/detail/cas64strong.hpp>
2008
#endif
2009

    
2010
/* pull in 128-bit atomic type using cmpxchg16b above */
2011
#if defined(BOOST_ATOMIC_INT128_LOCK_FREE) && BOOST_ATOMIC_INT128_LOCK_FREE > 0
2012
#include <boost/atomic/detail/cas128strong.hpp>
2013
#endif
2014

    
2015
#endif /* !defined(BOOST_ATOMIC_FORCE_FALLBACK) */
2016

    
2017
#endif