Statistics
| Revision:

root / tmp / org.txm.statsengine.r.core.win32 / res / win32 / library / BH / include / boost / atomic / detail / ops_gcc_ppc.hpp @ 2486

History | View | Annotate | Download (24.4 kB)

1
/*
2
 * Distributed under the Boost Software License, Version 1.0.
3
 * (See accompanying file LICENSE_1_0.txt or copy at
4
 * http://www.boost.org/LICENSE_1_0.txt)
5
 *
6
 * Copyright (c) 2009 Helge Bahmann
7
 * Copyright (c) 2013 Tim Blechmann
8
 * Copyright (c) 2014 Andrey Semashev
9
 */
10
/*!
11
 * \file   atomic/detail/ops_gcc_ppc.hpp
12
 *
13
 * This header contains implementation of the \c operations template.
14
 */
15

    
16
#ifndef BOOST_ATOMIC_DETAIL_OPS_GCC_PPC_HPP_INCLUDED_
17
#define BOOST_ATOMIC_DETAIL_OPS_GCC_PPC_HPP_INCLUDED_
18

    
19
#include <boost/memory_order.hpp>
20
#include <boost/atomic/detail/config.hpp>
21
#include <boost/atomic/detail/storage_type.hpp>
22
#include <boost/atomic/detail/operations_fwd.hpp>
23
#include <boost/atomic/capabilities.hpp>
24

    
25
#ifdef BOOST_HAS_PRAGMA_ONCE
26
#pragma once
27
#endif
28

    
29
namespace boost {
30
namespace atomics {
31
namespace detail {
32

    
33
// The implementation below uses information from this document:
34
// http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2010.02.19a.html
35

    
36
/*
37
    Refer to: Motorola: "Programming Environments Manual for 32-Bit
38
    Implementations of the PowerPC Architecture", Appendix E:
39
    "Synchronization Programming Examples" for an explanation of what is
40
    going on here (can be found on the web at various places by the
41
    name "MPCFPE32B.pdf", Google is your friend...)
42

43
    Most of the atomic operations map to instructions in a relatively
44
    straight-forward fashion, but "load"s may at first glance appear
45
    a bit strange as they map to:
46

47
            lwz %rX, addr
48
            cmpw %rX, %rX
49
            bne- 1f
50
        1:
51

52
    That is, the CPU is forced to perform a branch that "formally" depends
53
    on the value retrieved from memory. This scheme has an overhead of
54
    about 1-2 clock cycles per load, but it allows to map "acquire" to
55
    the "isync" instruction instead of "sync" uniformly and for all type
56
    of atomic operations. Since "isync" has a cost of about 15 clock
57
    cycles, while "sync" hast a cost of about 50 clock cycles, the small
58
    penalty to atomic loads more than compensates for this.
59

60
    Byte- and halfword-sized atomic values are realized by encoding the
61
    value to be represented into a word, performing sign/zero extension
62
    as appropriate. This means that after add/sub operations the value
63
    needs fixing up to accurately preserve the wrap-around semantic of
64
    the smaller type. (Nothing special needs to be done for the bit-wise
65
    and the "exchange type" operators as the compiler already sees to
66
    it that values carried in registers are extended appropriately and
67
    everything falls into place naturally).
68

69
    The register constraint "b"  instructs gcc to use any register
70
    except r0; this is sometimes required because the encoding for
71
    r0 is used to signify "constant zero" in a number of instructions,
72
    making r0 unusable in this place. For simplicity this constraint
73
    is used everywhere since I am to lazy to look this up on a
74
    per-instruction basis, and ppc has enough registers for this not
75
    to pose a problem.
76
*/
77

    
78
// A note about memory_order_consume. Technically, this architecture allows to avoid
79
// unnecessary memory barrier after consume load since it supports data dependency ordering.
80
// However, some compiler optimizations may break a seemingly valid code relying on data
81
// dependency tracking by injecting bogus branches to aid out of order execution.
82
// This may happen not only in Boost.Atomic code but also in user's code, which we have no
83
// control of. See this thread: http://lists.boost.org/Archives/boost/2014/06/213890.php.
84
// For this reason we promote memory_order_consume to memory_order_acquire.
85

    
86
struct gcc_ppc_operations_base
87
{
88
    static BOOST_FORCEINLINE void fence_before(memory_order order) BOOST_NOEXCEPT
89
    {
90
#if defined(__powerpc64__) || defined(__PPC64__)
91
        if (order == memory_order_seq_cst)
92
            __asm__ __volatile__ ("sync" ::: "memory");
93
        else if ((order & memory_order_release) != 0)
94
            __asm__ __volatile__ ("lwsync" ::: "memory");
95
#else
96
        if ((order & memory_order_release) != 0)
97
            __asm__ __volatile__ ("sync" ::: "memory");
98
#endif
99
    }
100

    
101
    static BOOST_FORCEINLINE void fence_after(memory_order order) BOOST_NOEXCEPT
102
    {
103
        if ((order & (memory_order_consume | memory_order_acquire)) != 0)
104
            __asm__ __volatile__ ("isync" ::: "memory");
105
    }
106
};
107

    
108

    
109
template< bool Signed >
110
struct operations< 4u, Signed > :
111
    public gcc_ppc_operations_base
112
{
113
    typedef typename make_storage_type< 4u, Signed >::type storage_type;
114
    typedef typename make_storage_type< 4u, Signed >::aligned aligned_storage_type;
115

    
116
    static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
117
    {
118
        fence_before(order);
119
        __asm__ __volatile__
120
        (
121
            "stw %1, %0\n\t"
122
            : "+m" (storage)
123
            : "r" (v)
124
        );
125
    }
126

    
127
    static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
128
    {
129
        storage_type v;
130
        if (order == memory_order_seq_cst)
131
            __asm__ __volatile__ ("sync" ::: "memory");
132
        if ((order & (memory_order_consume | memory_order_acquire)) != 0)
133
        {
134
            __asm__ __volatile__
135
            (
136
                "lwz %0, %1\n\t"
137
                "cmpw %0, %0\n\t"
138
                "bne- 1f\n\t"
139
                "1:\n\t"
140
                "isync\n\t"
141
                : "=&r" (v)
142
                : "m" (storage)
143
                : "cr0", "memory"
144
            );
145
        }
146
        else
147
        {
148
            __asm__ __volatile__
149
            (
150
                "lwz %0, %1\n\t"
151
                : "=&r" (v)
152
                : "m" (storage)
153
            );
154
        }
155
        return v;
156
    }
157

    
158
    static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
159
    {
160
        storage_type original;
161
        fence_before(order);
162
        __asm__ __volatile__
163
        (
164
            "1:\n\t"
165
            "lwarx %0,%y1\n\t"
166
            "stwcx. %2,%y1\n\t"
167
            "bne- 1b\n\t"
168
            : "=&b" (original), "+Z" (storage)
169
            : "b" (v)
170
            : "cr0"
171
        );
172
        fence_after(order);
173
        return original;
174
    }
175

    
176
    static BOOST_FORCEINLINE bool compare_exchange_weak(
177
        storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
178
    {
179
        int success;
180
        fence_before(success_order);
181
        __asm__ __volatile__
182
        (
183
            "li %1, 0\n\t"
184
            "lwarx %0,%y2\n\t"
185
            "cmpw %0, %3\n\t"
186
            "bne- 1f\n\t"
187
            "stwcx. %4,%y2\n\t"
188
            "bne- 1f\n\t"
189
            "li %1, 1\n\t"
190
            "1:\n\t"
191
            : "=&b" (expected), "=&b" (success), "+Z" (storage)
192
            : "b" (expected), "b" (desired)
193
            : "cr0"
194
        );
195
        if (success)
196
            fence_after(success_order);
197
        else
198
            fence_after(failure_order);
199
        return !!success;
200
    }
201

    
202
    static BOOST_FORCEINLINE bool compare_exchange_strong(
203
        storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
204
    {
205
        int success;
206
        fence_before(success_order);
207
        __asm__ __volatile__
208
        (
209
            "li %1, 0\n\t"
210
            "0: lwarx %0,%y2\n\t"
211
            "cmpw %0, %3\n\t"
212
            "bne- 1f\n\t"
213
            "stwcx. %4,%y2\n\t"
214
            "bne- 0b\n\t"
215
            "li %1, 1\n\t"
216
            "1:\n\t"
217
            : "=&b" (expected), "=&b" (success), "+Z" (storage)
218
            : "b" (expected), "b" (desired)
219
            : "cr0"
220
        );
221
        if (success)
222
            fence_after(success_order);
223
        else
224
            fence_after(failure_order);
225
        return !!success;
226
    }
227

    
228
    static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
229
    {
230
        storage_type original, tmp;
231
        fence_before(order);
232
        __asm__ __volatile__
233
        (
234
            "1:\n\t"
235
            "lwarx %0,%y2\n\t"
236
            "add %1,%0,%3\n\t"
237
            "stwcx. %1,%y2\n\t"
238
            "bne- 1b\n\t"
239
            : "=&b" (original), "=&b" (tmp), "+Z" (storage)
240
            : "b" (v)
241
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
242
        );
243
        fence_after(order);
244
        return original;
245
    }
246

    
247
    static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
248
    {
249
        storage_type original, tmp;
250
        fence_before(order);
251
        __asm__ __volatile__
252
        (
253
            "1:\n\t"
254
            "lwarx %0,%y2\n\t"
255
            "sub %1,%0,%3\n\t"
256
            "stwcx. %1,%y2\n\t"
257
            "bne- 1b\n\t"
258
            : "=&b" (original), "=&b" (tmp), "+Z" (storage)
259
            : "b" (v)
260
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
261
        );
262
        fence_after(order);
263
        return original;
264
    }
265

    
266
    static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
267
    {
268
        storage_type original, tmp;
269
        fence_before(order);
270
        __asm__ __volatile__
271
        (
272
            "1:\n\t"
273
            "lwarx %0,%y2\n\t"
274
            "and %1,%0,%3\n\t"
275
            "stwcx. %1,%y2\n\t"
276
            "bne- 1b\n\t"
277
            : "=&b" (original), "=&b" (tmp), "+Z" (storage)
278
            : "b" (v)
279
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
280
        );
281
        fence_after(order);
282
        return original;
283
    }
284

    
285
    static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
286
    {
287
        storage_type original, tmp;
288
        fence_before(order);
289
        __asm__ __volatile__
290
        (
291
            "1:\n\t"
292
            "lwarx %0,%y2\n\t"
293
            "or %1,%0,%3\n\t"
294
            "stwcx. %1,%y2\n\t"
295
            "bne- 1b\n\t"
296
            : "=&b" (original), "=&b" (tmp), "+Z" (storage)
297
            : "b" (v)
298
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
299
        );
300
        fence_after(order);
301
        return original;
302
    }
303

    
304
    static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
305
    {
306
        storage_type original, tmp;
307
        fence_before(order);
308
        __asm__ __volatile__
309
        (
310
            "1:\n\t"
311
            "lwarx %0,%y2\n\t"
312
            "xor %1,%0,%3\n\t"
313
            "stwcx. %1,%y2\n\t"
314
            "bne- 1b\n\t"
315
            : "=&b" (original), "=&b" (tmp), "+Z" (storage)
316
            : "b" (v)
317
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
318
        );
319
        fence_after(order);
320
        return original;
321
    }
322

    
323
    static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
324
    {
325
        return !!exchange(storage, (storage_type)1, order);
326
    }
327

    
328
    static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
329
    {
330
        store(storage, 0, order);
331
    }
332

    
333
    static BOOST_FORCEINLINE bool is_lock_free(storage_type const volatile&) BOOST_NOEXCEPT
334
    {
335
        return true;
336
    }
337
};
338

    
339

    
340
template< >
341
struct operations< 1u, false > :
342
    public operations< 4u, false >
343
{
344
    typedef operations< 4u, false > base_type;
345
    typedef base_type::storage_type storage_type;
346

    
347
    static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
348
    {
349
        storage_type original, tmp;
350
        fence_before(order);
351
        __asm__ __volatile__
352
        (
353
            "1:\n\t"
354
            "lwarx %0,%y2\n\t"
355
            "add %1,%0,%3\n\t"
356
            "rlwinm %1, %1, 0, 0xff\n\t"
357
            "stwcx. %1,%y2\n\t"
358
            "bne- 1b\n\t"
359
            : "=&b" (original), "=&b" (tmp), "+Z" (storage)
360
            : "b" (v)
361
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
362
        );
363
        fence_after(order);
364
        return original;
365
    }
366

    
367
    static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
368
    {
369
        storage_type original, tmp;
370
        fence_before(order);
371
        __asm__ __volatile__
372
        (
373
            "1:\n\t"
374
            "lwarx %0,%y2\n\t"
375
            "sub %1,%0,%3\n\t"
376
            "rlwinm %1, %1, 0, 0xff\n\t"
377
            "stwcx. %1,%y2\n\t"
378
            "bne- 1b\n\t"
379
            : "=&b" (original), "=&b" (tmp), "+Z" (storage)
380
            : "b" (v)
381
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
382
        );
383
        fence_after(order);
384
        return original;
385
    }
386
};
387

    
388
template< >
389
struct operations< 1u, true > :
390
    public operations< 4u, true >
391
{
392
    typedef operations< 4u, true > base_type;
393
    typedef base_type::storage_type storage_type;
394

    
395
    static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
396
    {
397
        storage_type original, tmp;
398
        fence_before(order);
399
        __asm__ __volatile__
400
        (
401
            "1:\n\t"
402
            "lwarx %0,%y2\n\t"
403
            "add %1,%0,%3\n\t"
404
            "extsb %1, %1\n\t"
405
            "stwcx. %1,%y2\n\t"
406
            "bne- 1b\n\t"
407
            : "=&b" (original), "=&b" (tmp), "+Z" (storage)
408
            : "b" (v)
409
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
410
        );
411
        fence_after(order);
412
        return original;
413
    }
414

    
415
    static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
416
    {
417
        storage_type original, tmp;
418
        fence_before(order);
419
        __asm__ __volatile__
420
        (
421
            "1:\n\t"
422
            "lwarx %0,%y2\n\t"
423
            "sub %1,%0,%3\n\t"
424
            "extsb %1, %1\n\t"
425
            "stwcx. %1,%y2\n\t"
426
            "bne- 1b\n\t"
427
            : "=&b" (original), "=&b" (tmp), "+Z" (storage)
428
            : "b" (v)
429
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
430
        );
431
        fence_after(order);
432
        return original;
433
    }
434
};
435

    
436

    
437
template< >
438
struct operations< 2u, false > :
439
    public operations< 4u, false >
440
{
441
    typedef operations< 4u, false > base_type;
442
    typedef base_type::storage_type storage_type;
443

    
444
    static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
445
    {
446
        storage_type original, tmp;
447
        fence_before(order);
448
        __asm__ __volatile__
449
        (
450
            "1:\n\t"
451
            "lwarx %0,%y2\n\t"
452
            "add %1,%0,%3\n\t"
453
            "rlwinm %1, %1, 0, 0xffff\n\t"
454
            "stwcx. %1,%y2\n\t"
455
            "bne- 1b\n\t"
456
            : "=&b" (original), "=&b" (tmp), "+Z" (storage)
457
            : "b" (v)
458
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
459
        );
460
        fence_after(order);
461
        return original;
462
    }
463

    
464
    static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
465
    {
466
        storage_type original, tmp;
467
        fence_before(order);
468
        __asm__ __volatile__
469
        (
470
            "1:\n\t"
471
            "lwarx %0,%y2\n\t"
472
            "sub %1,%0,%3\n\t"
473
            "rlwinm %1, %1, 0, 0xffff\n\t"
474
            "stwcx. %1,%y2\n\t"
475
            "bne- 1b\n\t"
476
            : "=&b" (original), "=&b" (tmp), "+Z" (storage)
477
            : "b" (v)
478
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
479
        );
480
        fence_after(order);
481
        return original;
482
    }
483
};
484

    
485
template< >
486
struct operations< 2u, true > :
487
    public operations< 4u, true >
488
{
489
    typedef operations< 4u, true > base_type;
490
    typedef base_type::storage_type storage_type;
491

    
492
    static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
493
    {
494
        storage_type original, tmp;
495
        fence_before(order);
496
        __asm__ __volatile__
497
        (
498
            "1:\n\t"
499
            "lwarx %0,%y2\n\t"
500
            "add %1,%0,%3\n\t"
501
            "extsh %1, %1\n\t"
502
            "stwcx. %1,%y2\n\t"
503
            "bne- 1b\n\t"
504
            : "=&b" (original), "=&b" (tmp), "+Z" (storage)
505
            : "b" (v)
506
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
507
        );
508
        fence_after(order);
509
        return original;
510
    }
511

    
512
    static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
513
    {
514
        storage_type original, tmp;
515
        fence_before(order);
516
        __asm__ __volatile__
517
        (
518
            "1:\n\t"
519
            "lwarx %0,%y2\n\t"
520
            "sub %1,%0,%3\n\t"
521
            "extsh %1, %1\n\t"
522
            "stwcx. %1,%y2\n\t"
523
            "bne- 1b\n\t"
524
            : "=&b" (original), "=&b" (tmp), "+Z" (storage)
525
            : "b" (v)
526
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
527
        );
528
        fence_after(order);
529
        return original;
530
    }
531
};
532

    
533

    
534
#if defined(__powerpc64__) || defined(__PPC64__)
535

    
536
template< bool Signed >
537
struct operations< 8u, Signed > :
538
    public gcc_ppc_operations_base
539
{
540
    typedef typename make_storage_type< 8u, Signed >::type storage_type;
541
    typedef typename make_storage_type< 8u, Signed >::aligned aligned_storage_type;
542

    
543
    static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
544
    {
545
        fence_before(order);
546
        __asm__ __volatile__
547
        (
548
            "std %1, %0\n\t"
549
            : "+m" (storage)
550
            : "r" (v)
551
        );
552
    }
553

    
554
    static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
555
    {
556
        storage_type v;
557
        if (order == memory_order_seq_cst)
558
            __asm__ __volatile__ ("sync" ::: "memory");
559
        if ((order & (memory_order_consume | memory_order_acquire)) != 0)
560
        {
561
            __asm__ __volatile__
562
            (
563
                "ld %0, %1\n\t"
564
                "cmpd %0, %0\n\t"
565
                "bne- 1f\n\t"
566
                "1:\n\t"
567
                "isync\n\t"
568
                : "=&b" (v)
569
                : "m" (storage)
570
                : "cr0", "memory"
571
            );
572
        }
573
        else
574
        {
575
            __asm__ __volatile__
576
            (
577
                "ld %0, %1\n\t"
578
                : "=&b" (v)
579
                : "m" (storage)
580
            );
581
        }
582
        return v;
583
    }
584

    
585
    static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
586
    {
587
        storage_type original;
588
        fence_before(order);
589
        __asm__ __volatile__
590
        (
591
            "1:\n\t"
592
            "ldarx %0,%y1\n\t"
593
            "stdcx. %2,%y1\n\t"
594
            "bne- 1b\n\t"
595
            : "=&b" (original), "+Z" (storage)
596
            : "b" (v)
597
            : "cr0"
598
        );
599
        fence_after(order);
600
        return original;
601
    }
602

    
603
    static BOOST_FORCEINLINE bool compare_exchange_weak(
604
        storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
605
    {
606
        int success;
607
        fence_before(success_order);
608
        __asm__ __volatile__
609
        (
610
            "li %1, 0\n\t"
611
            "ldarx %0,%y2\n\t"
612
            "cmpd %0, %3\n\t"
613
            "bne- 1f\n\t"
614
            "stdcx. %4,%y2\n\t"
615
            "bne- 1f\n\t"
616
            "li %1, 1\n\t"
617
            "1:"
618
            : "=&b" (expected), "=&b" (success), "+Z" (storage)
619
            : "b" (expected), "b" (desired)
620
            : "cr0"
621
        );
622
        if (success)
623
            fence_after(success_order);
624
        else
625
            fence_after(failure_order);
626
        return !!success;
627
    }
628

    
629
    static BOOST_FORCEINLINE bool compare_exchange_strong(
630
        storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
631
    {
632
        int success;
633
        fence_before(success_order);
634
        __asm__ __volatile__
635
        (
636
            "li %1, 0\n\t"
637
            "0: ldarx %0,%y2\n\t"
638
            "cmpd %0, %3\n\t"
639
            "bne- 1f\n\t"
640
            "stdcx. %4,%y2\n\t"
641
            "bne- 0b\n\t"
642
            "li %1, 1\n\t"
643
            "1:\n\t"
644
            : "=&b" (expected), "=&b" (success), "+Z" (storage)
645
            : "b" (expected), "b" (desired)
646
            : "cr0"
647
        );
648
        if (success)
649
            fence_after(success_order);
650
        else
651
            fence_after(failure_order);
652
        return !!success;
653
    }
654

    
655
    static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
656
    {
657
        storage_type original, tmp;
658
        fence_before(order);
659
        __asm__ __volatile__
660
        (
661
            "1:\n\t"
662
            "ldarx %0,%y2\n\t"
663
            "add %1,%0,%3\n\t"
664
            "stdcx. %1,%y2\n\t"
665
            "bne- 1b\n\t"
666
            : "=&b" (original), "=&b" (tmp), "+Z" (storage)
667
            : "b" (v)
668
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
669
        );
670
        fence_after(order);
671
        return original;
672
    }
673

    
674
    static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
675
    {
676
        storage_type original, tmp;
677
        fence_before(order);
678
        __asm__ __volatile__
679
        (
680
            "1:\n\t"
681
            "ldarx %0,%y2\n\t"
682
            "sub %1,%0,%3\n\t"
683
            "stdcx. %1,%y2\n\t"
684
            "bne- 1b\n\t"
685
            : "=&b" (original), "=&b" (tmp), "+Z" (storage)
686
            : "b" (v)
687
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
688
        );
689
        fence_after(order);
690
        return original;
691
    }
692

    
693
    static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
694
    {
695
        storage_type original, tmp;
696
        fence_before(order);
697
        __asm__ __volatile__
698
        (
699
            "1:\n\t"
700
            "ldarx %0,%y2\n\t"
701
            "and %1,%0,%3\n\t"
702
            "stdcx. %1,%y2\n\t"
703
            "bne- 1b\n\t"
704
            : "=&b" (original), "=&b" (tmp), "+Z" (storage)
705
            : "b" (v)
706
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
707
        );
708
        fence_after(order);
709
        return original;
710
    }
711

    
712
    static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
713
    {
714
        storage_type original, tmp;
715
        fence_before(order);
716
        __asm__ __volatile__
717
        (
718
            "1:\n\t"
719
            "ldarx %0,%y2\n\t"
720
            "or %1,%0,%3\n\t"
721
            "stdcx. %1,%y2\n\t"
722
            "bne- 1b\n\t"
723
            : "=&b" (original), "=&b" (tmp), "+Z" (storage)
724
            : "b" (v)
725
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
726
        );
727
        fence_after(order);
728
        return original;
729
    }
730

    
731
    static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
732
    {
733
        storage_type original, tmp;
734
        fence_before(order);
735
        __asm__ __volatile__
736
        (
737
            "1:\n\t"
738
            "ldarx %0,%y2\n\t"
739
            "xor %1,%0,%3\n\t"
740
            "stdcx. %1,%y2\n\t"
741
            "bne- 1b\n\t"
742
            : "=&b" (original), "=&b" (tmp), "+Z" (storage)
743
            : "b" (v)
744
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
745
        );
746
        fence_after(order);
747
        return original;
748
    }
749

    
750
    static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
751
    {
752
        return !!exchange(storage, (storage_type)1, order);
753
    }
754

    
755
    static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
756
    {
757
        store(storage, 0, order);
758
    }
759

    
760
    static BOOST_FORCEINLINE bool is_lock_free(storage_type const volatile&) BOOST_NOEXCEPT
761
    {
762
        return true;
763
    }
764
};
765

    
766
#endif // defined(__powerpc64__) || defined(__PPC64__)
767

    
768

    
769
BOOST_FORCEINLINE void thread_fence(memory_order order) BOOST_NOEXCEPT
770
{
771
    switch (order)
772
    {
773
    case memory_order_consume:
774
    case memory_order_acquire:
775
    case memory_order_release:
776
    case memory_order_acq_rel:
777
#if defined(__powerpc64__) || defined(__PPC64__)
778
        __asm__ __volatile__ ("lwsync" ::: "memory");
779
        break;
780
#endif
781
    case memory_order_seq_cst:
782
        __asm__ __volatile__ ("sync" ::: "memory");
783
        break;
784
    default:;
785
    }
786
}
787

    
788
BOOST_FORCEINLINE void signal_fence(memory_order order) BOOST_NOEXCEPT
789
{
790
    if (order != memory_order_relaxed)
791
#if defined(__ibmxl__) || defined(__IBMCPP__)
792
        __fence();
793
#else
794
        __asm__ __volatile__ ("" ::: "memory");
795
#endif
796
}
797

    
798
} // namespace detail
799
} // namespace atomics
800
} // namespace boost
801

    
802
#endif // BOOST_ATOMIC_DETAIL_OPS_GCC_PPC_HPP_INCLUDED_