Statistics
| Revision:

root / tmp / org.txm.statsengine.r.core.win32 / res / win32 / library / BH / include / boost / atomic / detail / ops_gcc_arm.hpp @ 2486

History | View | Annotate | Download (41.9 kB)

1
/*
2
 * Distributed under the Boost Software License, Version 1.0.
3
 * (See accompanying file LICENSE_1_0.txt or copy at
4
 * http://www.boost.org/LICENSE_1_0.txt)
5
 *
6
 * Copyright (c) 2009 Helge Bahmann
7
 * Copyright (c) 2013 Tim Blechmann
8
 * Copyright (c) 2014 Andrey Semashev
9
 */
10
/*!
11
 * \file   atomic/detail/ops_gcc_arm.hpp
12
 *
13
 * This header contains implementation of the \c operations template.
14
 */
15

    
16
#ifndef BOOST_ATOMIC_DETAIL_OPS_GCC_ARM_HPP_INCLUDED_
17
#define BOOST_ATOMIC_DETAIL_OPS_GCC_ARM_HPP_INCLUDED_
18

    
19
#include <boost/cstdint.hpp>
20
#include <boost/memory_order.hpp>
21
#include <boost/atomic/detail/config.hpp>
22
#include <boost/atomic/detail/storage_type.hpp>
23
#include <boost/atomic/detail/operations_fwd.hpp>
24
#include <boost/atomic/detail/ops_extending_cas_based.hpp>
25
#include <boost/atomic/capabilities.hpp>
26

    
27
#ifdef BOOST_HAS_PRAGMA_ONCE
28
#pragma once
29
#endif
30

    
31
namespace boost {
32
namespace atomics {
33
namespace detail {
34

    
35
// From the ARM Architecture Reference Manual for architecture v6:
36
//
37
// LDREX{<cond>} <Rd>, [<Rn>]
38
// <Rd> Specifies the destination register for the memory word addressed by <Rd>
39
// <Rn> Specifies the register containing the address.
40
//
41
// STREX{<cond>} <Rd>, <Rm>, [<Rn>]
42
// <Rd> Specifies the destination register for the returned status value.
43
//      0  if the operation updates memory
44
//      1  if the operation fails to update memory
45
// <Rm> Specifies the register containing the word to be stored to memory.
46
// <Rn> Specifies the register containing the address.
47
// Rd must not be the same register as Rm or Rn.
48
//
49
// ARM v7 is like ARM v6 plus:
50
// There are half-word and byte versions of the LDREX and STREX instructions,
51
// LDREXH, LDREXB, STREXH and STREXB.
52
// There are also double-word versions, LDREXD and STREXD.
53
// (Actually it looks like these are available from version 6k onwards.)
54
// FIXME these are not yet used; should be mostly a matter of copy-and-paste.
55
// I think you can supply an immediate offset to the address.
56
//
57
// A memory barrier is effected using a "co-processor 15" instruction,
58
// though a separate assembler mnemonic is available for it in v7.
59
//
60
// "Thumb 1" is a subset of the ARM instruction set that uses a 16-bit encoding.  It
61
// doesn't include all instructions and in particular it doesn't include the co-processor
62
// instruction used for the memory barrier or the load-locked/store-conditional
63
// instructions.  So, if we're compiling in "Thumb 1" mode, we need to wrap all of our
64
// asm blocks with code to temporarily change to ARM mode.
65
//
66
// You can only change between ARM and Thumb modes when branching using the bx instruction.
67
// bx takes an address specified in a register.  The least significant bit of the address
68
// indicates the mode, so 1 is added to indicate that the destination code is Thumb.
69
// A temporary register is needed for the address and is passed as an argument to these
70
// macros.  It must be one of the "low" registers accessible to Thumb code, specified
71
// using the "l" attribute in the asm statement.
72
//
73
// Architecture v7 introduces "Thumb 2", which does include (almost?) all of the ARM
74
// instruction set.  (Actually, there was an extension of v6 called v6T2 which supported
75
// "Thumb 2" mode, but its architecture manual is no longer available, referring to v7.)
76
// So in v7 we don't need to change to ARM mode; we can write "universal
77
// assembler" which will assemble to Thumb 2 or ARM code as appropriate.  The only thing
78
// we need to do to make this "universal" assembler mode work is to insert "IT" instructions
79
// to annotate the conditional instructions.  These are ignored in other modes (e.g. v6),
80
// so they can always be present.
81

    
82
// A note about memory_order_consume. Technically, this architecture allows to avoid
83
// unnecessary memory barrier after consume load since it supports data dependency ordering.
84
// However, some compiler optimizations may break a seemingly valid code relying on data
85
// dependency tracking by injecting bogus branches to aid out of order execution.
86
// This may happen not only in Boost.Atomic code but also in user's code, which we have no
87
// control of. See this thread: http://lists.boost.org/Archives/boost/2014/06/213890.php.
88
// For this reason we promote memory_order_consume to memory_order_acquire.
89

    
90
#if defined(__thumb__) && !defined(__thumb2__)
91
#define BOOST_ATOMIC_DETAIL_ARM_ASM_START(TMPREG) "adr " #TMPREG ", 8f\n" "bx " #TMPREG "\n" ".arm\n" ".align 4\n" "8:\n"
92
#define BOOST_ATOMIC_DETAIL_ARM_ASM_END(TMPREG)   "adr " #TMPREG ", 9f + 1\n" "bx " #TMPREG "\n" ".thumb\n" ".align 2\n" "9:\n"
93
#define BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(var) "=&l" (var)
94
#else
95
// The tmpreg may be wasted in this case, which is non-optimal.
96
#define BOOST_ATOMIC_DETAIL_ARM_ASM_START(TMPREG)
97
#define BOOST_ATOMIC_DETAIL_ARM_ASM_END(TMPREG)
98
#define BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(var) "=&r" (var)
99
#endif
100

    
101
struct gcc_arm_operations_base
102
{
103
    static BOOST_FORCEINLINE void fence_before(memory_order order) BOOST_NOEXCEPT
104
    {
105
        if ((order & memory_order_release) != 0)
106
            hardware_full_fence();
107
    }
108

    
109
    static BOOST_FORCEINLINE void fence_after(memory_order order) BOOST_NOEXCEPT
110
    {
111
        if ((order & (memory_order_consume | memory_order_acquire)) != 0)
112
            hardware_full_fence();
113
    }
114

    
115
    static BOOST_FORCEINLINE void fence_after_store(memory_order order) BOOST_NOEXCEPT
116
    {
117
        if (order == memory_order_seq_cst)
118
            hardware_full_fence();
119
    }
120

    
121
    static BOOST_FORCEINLINE void hardware_full_fence() BOOST_NOEXCEPT
122
    {
123
#if defined(BOOST_ATOMIC_DETAIL_ARM_HAS_DMB)
124
        // Older binutils (supposedly, older than 2.21.1) didn't support symbolic or numeric arguments of the "dmb" instruction such as "ish" or "#11".
125
        // As a workaround we have to inject encoded bytes of the instruction. There are two encodings for the instruction: ARM and Thumb. See ARM Architecture Reference Manual, A8.8.43.
126
        // Since we cannot detect binutils version at compile time, we'll have to always use this hack.
127
        __asm__ __volatile__
128
        (
129
#if defined(__thumb2__)
130
            ".short 0xF3BF, 0x8F5B\n" // dmb ish
131
#else
132
            ".word 0xF57FF05B\n" // dmb ish
133
#endif
134
            :
135
            :
136
            : "memory"
137
        );
138
#else
139
        int tmp;
140
        __asm__ __volatile__
141
        (
142
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
143
            "mcr\tp15, 0, r0, c7, c10, 5\n"
144
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
145
            : "=&l" (tmp)
146
            :
147
            : "memory"
148
        );
149
#endif
150
    }
151
};
152

    
153

    
154
template< bool Signed >
155
struct operations< 4u, Signed > :
156
    public gcc_arm_operations_base
157
{
158
    typedef typename make_storage_type< 4u, Signed >::type storage_type;
159
    typedef typename make_storage_type< 4u, Signed >::aligned aligned_storage_type;
160

    
161
    static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
162
    {
163
        fence_before(order);
164
        storage = v;
165
        fence_after_store(order);
166
    }
167

    
168
    static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
169
    {
170
        storage_type v = storage;
171
        fence_after(order);
172
        return v;
173
    }
174

    
175
    static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
176
    {
177
        storage_type original;
178
        fence_before(order);
179
        uint32_t tmp;
180
        __asm__ __volatile__
181
        (
182
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
183
            "1:\n"
184
            "ldrex %[original], %[storage]\n"          // load the original value
185
            "strex %[tmp], %[value], %[storage]\n"     // store the replacement, tmp = store failed
186
            "teq   %[tmp], #0\n"                       // check if store succeeded
187
            "bne   1b\n"
188
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
189
            : [tmp] "=&l" (tmp), [original] "=&r" (original), [storage] "+Q" (storage)
190
            : [value] "r" (v)
191
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
192
        );
193
        fence_after(order);
194
        return original;
195
    }
196

    
197
    static BOOST_FORCEINLINE bool compare_exchange_weak(
198
        storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
199
    {
200
        fence_before(success_order);
201
        uint32_t success;
202
        uint32_t tmp;
203
        storage_type original;
204
        __asm__ __volatile__
205
        (
206
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
207
            "mov     %[success], #0\n"                      // success = 0
208
            "ldrex   %[original], %[storage]\n"             // original = *(&storage)
209
            "cmp     %[original], %[expected]\n"            // flags = original==expected
210
            "itt     eq\n"                                  // [hint that the following 2 instructions are conditional on flags.equal]
211
            "strexeq %[success], %[desired], %[storage]\n"  // if (flags.equal) *(&storage) = desired, success = store failed
212
            "eoreq   %[success], %[success], #1\n"          // if (flags.equal) success ^= 1 (i.e. make it 1 if store succeeded)
213
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
214
            : [original] "=&r" (original),  // %0
215
              [success] "=&r" (success),    // %1
216
              [tmp] "=&l" (tmp),            // %2
217
              [storage] "+Q" (storage)      // %3
218
            : [expected] "r" (expected),    // %4
219
              [desired] "r" (desired)       // %5
220
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
221
        );
222
        if (success)
223
            fence_after(success_order);
224
        else
225
            fence_after(failure_order);
226
        expected = original;
227
        return !!success;
228
    }
229

    
230
    static BOOST_FORCEINLINE bool compare_exchange_strong(
231
        storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
232
    {
233
        fence_before(success_order);
234
        uint32_t success;
235
        uint32_t tmp;
236
        storage_type original;
237
        __asm__ __volatile__
238
        (
239
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
240
            "mov     %[success], #0\n"                      // success = 0
241
            "1:\n"
242
            "ldrex   %[original], %[storage]\n"             // original = *(&storage)
243
            "cmp     %[original], %[expected]\n"            // flags = original==expected
244
            "bne     2f\n"                                  // if (!flags.equal) goto end
245
            "strex   %[success], %[desired], %[storage]\n"  // *(&storage) = desired, success = store failed
246
            "eors    %[success], %[success], #1\n"          // success ^= 1 (i.e. make it 1 if store succeeded); flags.equal = success == 0
247
            "beq     1b\n"                                  // if (flags.equal) goto retry
248
            "2:\n"
249
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
250
            : [original] "=&r" (original),  // %0
251
              [success] "=&r" (success),    // %1
252
              [tmp] "=&l" (tmp),            // %2
253
              [storage] "+Q" (storage)      // %3
254
            : [expected] "r" (expected),    // %4
255
              [desired] "r" (desired)       // %5
256
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
257
        );
258
        if (success)
259
            fence_after(success_order);
260
        else
261
            fence_after(failure_order);
262
        expected = original;
263
        return !!success;
264
    }
265

    
266
    static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
267
    {
268
        fence_before(order);
269
        uint32_t tmp;
270
        storage_type original, result;
271
        __asm__ __volatile__
272
        (
273
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
274
            "1:\n"
275
            "ldrex   %[original], %[storage]\n"           // original = *(&storage)
276
            "add     %[result], %[original], %[value]\n"  // result = original + value
277
            "strex   %[tmp], %[result], %[storage]\n"     // *(&storage) = result, tmp = store failed
278
            "teq     %[tmp], #0\n"                        // flags = tmp==0
279
            "bne     1b\n"                                // if (!flags.equal) goto retry
280
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
281
            : [original] "=&r" (original),  // %0
282
              [result] "=&r" (result),      // %1
283
              [tmp] "=&l" (tmp),            // %2
284
              [storage] "+Q" (storage)      // %3
285
            : [value] "r" (v)               // %4
286
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
287
        );
288
        fence_after(order);
289
        return original;
290
    }
291

    
292
    static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
293
    {
294
        fence_before(order);
295
        uint32_t tmp;
296
        storage_type original, result;
297
        __asm__ __volatile__
298
        (
299
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
300
            "1:\n"
301
            "ldrex   %[original], %[storage]\n"           // original = *(&storage)
302
            "sub     %[result], %[original], %[value]\n"  // result = original - value
303
            "strex   %[tmp], %[result], %[storage]\n"     // *(&storage) = result, tmp = store failed
304
            "teq     %[tmp], #0\n"                        // flags = tmp==0
305
            "bne     1b\n"                                // if (!flags.equal) goto retry
306
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
307
            : [original] "=&r" (original),  // %0
308
              [result] "=&r" (result),      // %1
309
              [tmp] "=&l" (tmp),            // %2
310
              [storage] "+Q" (storage)      // %3
311
            : [value] "r" (v)               // %4
312
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
313
        );
314
        fence_after(order);
315
        return original;
316
    }
317

    
318
    static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
319
    {
320
        fence_before(order);
321
        uint32_t tmp;
322
        storage_type original, result;
323
        __asm__ __volatile__
324
        (
325
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
326
            "1:\n"
327
            "ldrex   %[original], %[storage]\n"           // original = *(&storage)
328
            "and     %[result], %[original], %[value]\n"  // result = original & value
329
            "strex   %[tmp], %[result], %[storage]\n"     // *(&storage) = result, tmp = store failed
330
            "teq     %[tmp], #0\n"                        // flags = tmp==0
331
            "bne     1b\n"                                // if (!flags.equal) goto retry
332
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
333
            : [original] "=&r" (original),  // %0
334
              [result] "=&r" (result),      // %1
335
              [tmp] "=&l" (tmp),            // %2
336
              [storage] "+Q" (storage)      // %3
337
            : [value] "r" (v)               // %4
338
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
339
        );
340
        fence_after(order);
341
        return original;
342
    }
343

    
344
    static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
345
    {
346
        fence_before(order);
347
        uint32_t tmp;
348
        storage_type original, result;
349
        __asm__ __volatile__
350
        (
351
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
352
            "1:\n"
353
            "ldrex   %[original], %[storage]\n"           // original = *(&storage)
354
            "orr     %[result], %[original], %[value]\n"  // result = original | value
355
            "strex   %[tmp], %[result], %[storage]\n"     // *(&storage) = result, tmp = store failed
356
            "teq     %[tmp], #0\n"                        // flags = tmp==0
357
            "bne     1b\n"                                // if (!flags.equal) goto retry
358
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
359
            : [original] "=&r" (original),  // %0
360
              [result] "=&r" (result),      // %1
361
              [tmp] "=&l" (tmp),            // %2
362
              [storage] "+Q" (storage)      // %3
363
            : [value] "r" (v)               // %4
364
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
365
        );
366
        fence_after(order);
367
        return original;
368
    }
369

    
370
    static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
371
    {
372
        fence_before(order);
373
        uint32_t tmp;
374
        storage_type original, result;
375
        __asm__ __volatile__
376
        (
377
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
378
            "1:\n"
379
            "ldrex   %[original], %[storage]\n"           // original = *(&storage)
380
            "eor     %[result], %[original], %[value]\n"  // result = original ^ value
381
            "strex   %[tmp], %[result], %[storage]\n"     // *(&storage) = result, tmp = store failed
382
            "teq     %[tmp], #0\n"                        // flags = tmp==0
383
            "bne     1b\n"                                // if (!flags.equal) goto retry
384
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
385
            : [original] "=&r" (original),  // %0
386
              [result] "=&r" (result),      // %1
387
              [tmp] "=&l" (tmp),            // %2
388
              [storage] "+Q" (storage)      // %3
389
            : [value] "r" (v)               // %4
390
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
391
        );
392
        fence_after(order);
393
        return original;
394
    }
395

    
396
    static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
397
    {
398
        return !!exchange(storage, (storage_type)1, order);
399
    }
400

    
401
    static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
402
    {
403
        store(storage, 0, order);
404
    }
405

    
406
    static BOOST_FORCEINLINE bool is_lock_free(storage_type const volatile&) BOOST_NOEXCEPT
407
    {
408
        return true;
409
    }
410
};
411

    
412

    
413
template< >
414
struct operations< 1u, false > :
415
    public operations< 4u, false >
416
{
417
    typedef operations< 4u, false > base_type;
418
    typedef base_type::storage_type storage_type;
419

    
420
    static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
421
    {
422
        fence_before(order);
423
        uint32_t tmp;
424
        storage_type original, result;
425
        __asm__ __volatile__
426
        (
427
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
428
            "1:\n"
429
            "ldrex   %[original], %[storage]\n"           // original = *(&storage)
430
            "add     %[result], %[original], %[value]\n"  // result = original + value
431
            "uxtb    %[result], %[result]\n"              // zero extend result from 8 to 32 bits
432
            "strex   %[tmp], %[result], %[storage]\n"     // *(&storage) = result, tmp = store failed
433
            "teq     %[tmp], #0\n"                        // flags = tmp==0
434
            "bne     1b\n"                                // if (!flags.equal) goto retry
435
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
436
            : [original] "=&r" (original),  // %0
437
              [result] "=&r" (result),      // %1
438
              [tmp] "=&l" (tmp),            // %2
439
              [storage] "+Q" (storage)      // %3
440
            : [value] "r" (v)               // %4
441
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
442
        );
443
        fence_after(order);
444
        return original;
445
    }
446

    
447
    static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
448
    {
449
        fence_before(order);
450
        uint32_t tmp;
451
        storage_type original, result;
452
        __asm__ __volatile__
453
        (
454
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
455
            "1:\n"
456
            "ldrex   %[original], %[storage]\n"           // original = *(&storage)
457
            "sub     %[result], %[original], %[value]\n"  // result = original - value
458
            "uxtb    %[result], %[result]\n"              // zero extend result from 8 to 32 bits
459
            "strex   %[tmp], %[result], %[storage]\n"     // *(&storage) = result, tmp = store failed
460
            "teq     %[tmp], #0\n"                        // flags = tmp==0
461
            "bne     1b\n"                                // if (!flags.equal) goto retry
462
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
463
            : [original] "=&r" (original),  // %0
464
              [result] "=&r" (result),      // %1
465
              [tmp] "=&l" (tmp),            // %2
466
              [storage] "+Q" (storage)      // %3
467
            : [value] "r" (v)               // %4
468
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
469
        );
470
        fence_after(order);
471
        return original;
472
    }
473
};
474

    
475
template< >
476
struct operations< 1u, true > :
477
    public operations< 4u, true >
478
{
479
    typedef operations< 4u, true > base_type;
480
    typedef base_type::storage_type storage_type;
481

    
482
    static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
483
    {
484
        fence_before(order);
485
        uint32_t tmp;
486
        storage_type original, result;
487
        __asm__ __volatile__
488
        (
489
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
490
            "1:\n"
491
            "ldrex   %[original], %[storage]\n"           // original = *(&storage)
492
            "add     %[result], %[original], %[value]\n"  // result = original + value
493
            "sxtb    %[result], %[result]\n"              // sign extend result from 8 to 32 bits
494
            "strex   %[tmp], %[result], %[storage]\n"     // *(&storage) = result, tmp = store failed
495
            "teq     %[tmp], #0\n"                        // flags = tmp==0
496
            "bne     1b\n"                                // if (!flags.equal) goto retry
497
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
498
            : [original] "=&r" (original),  // %0
499
              [result] "=&r" (result),      // %1
500
              [tmp] "=&l" (tmp),            // %2
501
              [storage] "+Q" (storage)      // %3
502
            : [value] "r" (v)               // %4
503
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
504
        );
505
        fence_after(order);
506
        return original;
507
    }
508

    
509
    static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
510
    {
511
        fence_before(order);
512
        uint32_t tmp;
513
        storage_type original, result;
514
        __asm__ __volatile__
515
        (
516
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
517
            "1:\n"
518
            "ldrex   %[original], %[storage]\n"           // original = *(&storage)
519
            "sub     %[result], %[original], %[value]\n"  // result = original - value
520
            "sxtb    %[result], %[result]\n"              // sign extend result from 8 to 32 bits
521
            "strex   %[tmp], %[result], %[storage]\n"     // *(&storage) = result, tmp = store failed
522
            "teq     %[tmp], #0\n"                        // flags = tmp==0
523
            "bne     1b\n"                                // if (!flags.equal) goto retry
524
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
525
            : [original] "=&r" (original),  // %0
526
              [result] "=&r" (result),      // %1
527
              [tmp] "=&l" (tmp),            // %2
528
              [storage] "+Q" (storage)      // %3
529
            : [value] "r" (v)               // %4
530
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
531
        );
532
        fence_after(order);
533
        return original;
534
    }
535
};
536

    
537

    
538
template< >
539
struct operations< 2u, false > :
540
    public operations< 4u, false >
541
{
542
    typedef operations< 4u, false > base_type;
543
    typedef base_type::storage_type storage_type;
544

    
545
    static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
546
    {
547
        fence_before(order);
548
        uint32_t tmp;
549
        storage_type original, result;
550
        __asm__ __volatile__
551
        (
552
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
553
            "1:\n"
554
            "ldrex   %[original], %[storage]\n"           // original = *(&storage)
555
            "add     %[result], %[original], %[value]\n"  // result = original + value
556
            "uxth    %[result], %[result]\n"              // zero extend result from 16 to 32 bits
557
            "strex   %[tmp], %[result], %[storage]\n"     // *(&storage) = result, tmp = store failed
558
            "teq     %[tmp], #0\n"                        // flags = tmp==0
559
            "bne     1b\n"                                // if (!flags.equal) goto retry
560
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
561
            : [original] "=&r" (original),  // %0
562
              [result] "=&r" (result),      // %1
563
              [tmp] "=&l" (tmp),            // %2
564
              [storage] "+Q" (storage)      // %3
565
            : [value] "r" (v)               // %4
566
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
567
        );
568
        fence_after(order);
569
        return original;
570
    }
571

    
572
    static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
573
    {
574
        fence_before(order);
575
        uint32_t tmp;
576
        storage_type original, result;
577
        __asm__ __volatile__
578
        (
579
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
580
            "1:\n"
581
            "ldrex   %[original], %[storage]\n"           // original = *(&storage)
582
            "sub     %[result], %[original], %[value]\n"  // result = original - value
583
            "uxth    %[result], %[result]\n"              // zero extend result from 16 to 32 bits
584
            "strex   %[tmp], %[result], %[storage]\n"     // *(&storage) = result, tmp = store failed
585
            "teq     %[tmp], #0\n"                        // flags = tmp==0
586
            "bne     1b\n"                                // if (!flags.equal) goto retry
587
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
588
            : [original] "=&r" (original),  // %0
589
              [result] "=&r" (result),      // %1
590
              [tmp] "=&l" (tmp),            // %2
591
              [storage] "+Q" (storage)      // %3
592
            : [value] "r" (v)               // %4
593
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
594
        );
595
        fence_after(order);
596
        return original;
597
    }
598
};
599

    
600
template< >
601
struct operations< 2u, true > :
602
    public operations< 4u, true >
603
{
604
    typedef operations< 4u, true > base_type;
605
    typedef base_type::storage_type storage_type;
606

    
607
    static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
608
    {
609
        fence_before(order);
610
        uint32_t tmp;
611
        storage_type original, result;
612
        __asm__ __volatile__
613
        (
614
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
615
            "1:\n"
616
            "ldrex   %[original], %[storage]\n"           // original = *(&storage)
617
            "add     %[result], %[original], %[value]\n"  // result = original + value
618
            "sxth    %[result], %[result]\n"              // sign extend result from 16 to 32 bits
619
            "strex   %[tmp], %[result], %[storage]\n"     // *(&storage) = result, tmp = store failed
620
            "teq     %[tmp], #0\n"                        // flags = tmp==0
621
            "bne     1b\n"                                // if (!flags.equal) goto retry
622
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
623
            : [original] "=&r" (original),  // %0
624
              [result] "=&r" (result),      // %1
625
              [tmp] "=&l" (tmp),            // %2
626
              [storage] "+Q" (storage)      // %3
627
            : [value] "r" (v)               // %4
628
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
629
        );
630
        fence_after(order);
631
        return original;
632
    }
633

    
634
    static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
635
    {
636
        fence_before(order);
637
        uint32_t tmp;
638
        storage_type original, result;
639
        __asm__ __volatile__
640
        (
641
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
642
            "1:\n"
643
            "ldrex   %[original], %[storage]\n"           // original = *(&storage)
644
            "sub     %[result], %[original], %[value]\n"  // result = original - value
645
            "sxth    %[result], %[result]\n"              // sign extend result from 16 to 32 bits
646
            "strex   %[tmp], %[result], %[storage]\n"     // *(&storage) = result, tmp = store failed
647
            "teq     %[tmp], #0\n"                        // flags = tmp==0
648
            "bne     1b\n"                                // if (!flags.equal) goto retry
649
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
650
            : [original] "=&r" (original),  // %0
651
              [result] "=&r" (result),      // %1
652
              [tmp] "=&l" (tmp),            // %2
653
              [storage] "+Q" (storage)      // %3
654
            : [value] "r" (v)               // %4
655
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
656
        );
657
        fence_after(order);
658
        return original;
659
    }
660
};
661

    
662

    
663
#if defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXD_STREXD)
664

    
665
// Unlike 32-bit operations, for 64-bit loads and stores we must use ldrexd/strexd.
666
// Any other instructions result in a non-atomic sequence of 32-bit accesses.
667
// See "ARM Architecture Reference Manual ARMv7-A and ARMv7-R edition",
668
// Section A3.5.3 "Atomicity in the ARM architecture".
669

    
670
// In the asm blocks below we have to use 32-bit register pairs to compose 64-bit values.
671
// In order to pass the 64-bit operands to/from asm blocks, we use undocumented gcc feature:
672
// the lower half (Rt) of the operand is accessible normally, via the numbered placeholder (e.g. %0),
673
// and the upper half (Rt2) - via the same placeholder with an 'H' after the '%' sign (e.g. %H0).
674
// See: http://hardwarebug.org/2010/07/06/arm-inline-asm-secrets/
675

    
676
template< bool Signed >
677
struct operations< 8u, Signed > :
678
    public gcc_arm_operations_base
679
{
680
    typedef typename make_storage_type< 8u, Signed >::type storage_type;
681
    typedef typename make_storage_type< 8u, Signed >::aligned aligned_storage_type;
682

    
683
    static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
684
    {
685
        exchange(storage, v, order);
686
    }
687

    
688
    static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
689
    {
690
        storage_type original;
691
        uint32_t tmp;
692
        __asm__ __volatile__
693
        (
694
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
695
            "ldrexd %1, %H1, [%2]\n"
696
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
697
            : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
698
              "=&r" (original)   // %1
699
            : "r" (&storage)     // %2
700
        );
701
        fence_after(order);
702
        return original;
703
    }
704

    
705
    static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
706
    {
707
        storage_type original;
708
        fence_before(order);
709
        uint32_t tmp;
710
        __asm__ __volatile__
711
        (
712
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
713
            "1:\n"
714
            "ldrexd %1, %H1, [%3]\n"        // load the original value
715
            "strexd %0, %2, %H2, [%3]\n"    // store the replacement, tmp = store failed
716
            "teq    %0, #0\n"               // check if store succeeded
717
            "bne    1b\n"
718
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
719
            : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
720
              "=&r" (original)   // %1
721
            : "r" (v),           // %2
722
              "r" (&storage)     // %3
723
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
724
        );
725
        fence_after(order);
726
        return original;
727
    }
728

    
729
    static BOOST_FORCEINLINE bool compare_exchange_weak(
730
        storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
731
    {
732
        fence_before(success_order);
733
        uint32_t tmp;
734
        storage_type original, old_val = expected;
735
        __asm__ __volatile__
736
        (
737
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
738
            "ldrexd   %1, %H1, [%3]\n"               // original = *(&storage)
739
            "cmp      %1, %2\n"                      // flags = original.lo==old_val.lo
740
            "ittt     eq\n"                          // [hint that the following 3 instructions are conditional on flags.equal]
741
            "cmpeq    %H1, %H2\n"                    // if (flags.equal) flags = original.hi==old_val.hi
742
            "strexdeq %0, %4, %H4, [%3]\n"           // if (flags.equal) *(&storage) = desired, tmp = store failed
743
            "teqeq    %0, #0\n"                      // if (flags.equal) flags = tmp==0
744
            "ite      eq\n"                          // [hint that the following 2 instructions are conditional on flags.equal]
745
            "moveq    %2, #1\n"                      // if (flags.equal) old_val.lo = 1
746
            "movne    %2, #0\n"                      // if (!flags.equal) old_val.lo = 0
747
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
748
            : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
749
              "=&r" (original),  // %1
750
              "+r" (old_val)     // %2
751
            : "r" (&storage),    // %3
752
              "r" (desired)      // %4
753
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
754
        );
755
        const uint32_t success = (uint32_t)old_val;
756
        if (success)
757
            fence_after(success_order);
758
        else
759
            fence_after(failure_order);
760
        expected = original;
761
        return !!success;
762
    }
763

    
764
    static BOOST_FORCEINLINE bool compare_exchange_strong(
765
        storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
766
    {
767
        fence_before(success_order);
768
        uint32_t tmp;
769
        storage_type original, old_val = expected;
770
        __asm__ __volatile__
771
        (
772
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
773
            "1:\n"
774
            "ldrexd  %1, %H1, [%3]\n"               // original = *(&storage)
775
            "cmp     %1, %2\n"                      // flags = original.lo==old_val.lo
776
            "it      eq\n"                          // [hint that the following instruction is conditional on flags.equal]
777
            "cmpeq   %H1, %H2\n"                    // if (flags.equal) flags = original.hi==old_val.hi
778
            "bne     2f\n"                          // if (!flags.equal) goto end
779
            "strexd  %0, %4, %H4, [%3]\n"           // *(&storage) = desired, tmp = store failed
780
            "teq     %0, #0\n"                      // flags.equal = tmp == 0
781
            "bne     1b\n"                          // if (flags.equal) goto retry
782
            "2:\n"
783
            "ite      eq\n"                         // [hint that the following 2 instructions are conditional on flags.equal]
784
            "moveq    %2, #1\n"                     // if (flags.equal) old_val.lo = 1
785
            "movne    %2, #0\n"                     // if (!flags.equal) old_val.lo = 0
786
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
787
            : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
788
              "=&r" (original),  // %1
789
              "+r" (old_val)     // %2
790
            : "r" (&storage),    // %3
791
              "r" (desired)      // %4
792
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
793
        );
794
        const uint32_t success = (uint32_t)old_val;
795
        if (success)
796
            fence_after(success_order);
797
        else
798
            fence_after(failure_order);
799
        expected = original;
800
        return !!success;
801
    }
802

    
803
    static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
804
    {
805
        fence_before(order);
806
        storage_type original, result;
807
        uint32_t tmp;
808
        __asm__ __volatile__
809
        (
810
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
811
            "1:\n"
812
            "ldrexd  %1, %H1, [%3]\n"               // original = *(&storage)
813
            "adds    %2, %1, %4\n"                  // result = original + value
814
            "adc     %H2, %H1, %H4\n"
815
            "strexd  %0, %2, %H2, [%3]\n"           // *(&storage) = result, tmp = store failed
816
            "teq     %0, #0\n"                      // flags = tmp==0
817
            "bne     1b\n"                          // if (!flags.equal) goto retry
818
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
819
            : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
820
              "=&r" (original),  // %1
821
              "=&r" (result)     // %2
822
            : "r" (&storage),    // %3
823
              "r" (v)            // %4
824
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
825
        );
826
        fence_after(order);
827
        return original;
828
    }
829

    
830
    static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
831
    {
832
        fence_before(order);
833
        storage_type original, result;
834
        uint32_t tmp;
835
        __asm__ __volatile__
836
        (
837
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
838
            "1:\n"
839
            "ldrexd  %1, %H1, [%3]\n"               // original = *(&storage)
840
            "subs    %2, %1, %4\n"                  // result = original - value
841
            "sbc     %H2, %H1, %H4\n"
842
            "strexd  %0, %2, %H2, [%3]\n"           // *(&storage) = result, tmp = store failed
843
            "teq     %0, #0\n"                      // flags = tmp==0
844
            "bne     1b\n"                          // if (!flags.equal) goto retry
845
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
846
            : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
847
              "=&r" (original),  // %1
848
              "=&r" (result)     // %2
849
            : "r" (&storage),    // %3
850
              "r" (v)            // %4
851
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
852
        );
853
        fence_after(order);
854
        return original;
855
    }
856

    
857
    static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
858
    {
859
        fence_before(order);
860
        storage_type original, result;
861
        uint32_t tmp;
862
        __asm__ __volatile__
863
        (
864
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
865
            "1:\n"
866
            "ldrexd  %1, %H1, [%3]\n"               // original = *(&storage)
867
            "and     %2, %1, %4\n"                  // result = original & value
868
            "and     %H2, %H1, %H4\n"
869
            "strexd  %0, %2, %H2, [%3]\n"           // *(&storage) = result, tmp = store failed
870
            "teq     %0, #0\n"                      // flags = tmp==0
871
            "bne     1b\n"                          // if (!flags.equal) goto retry
872
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
873
            : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
874
              "=&r" (original),  // %1
875
              "=&r" (result)     // %2
876
            : "r" (&storage),    // %3
877
              "r" (v)            // %4
878
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
879
        );
880
        fence_after(order);
881
        return original;
882
    }
883

    
884
    static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
885
    {
886
        fence_before(order);
887
        storage_type original, result;
888
        uint32_t tmp;
889
        __asm__ __volatile__
890
        (
891
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
892
            "1:\n"
893
            "ldrexd  %1, %H1, [%3]\n"               // original = *(&storage)
894
            "orr     %2, %1, %4\n"                  // result = original | value
895
            "orr     %H2, %H1, %H4\n"
896
            "strexd  %0, %2, %H2, [%3]\n"           // *(&storage) = result, tmp = store failed
897
            "teq     %0, #0\n"                      // flags = tmp==0
898
            "bne     1b\n"                          // if (!flags.equal) goto retry
899
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
900
            : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
901
              "=&r" (original),  // %1
902
              "=&r" (result)     // %2
903
            : "r" (&storage),    // %3
904
              "r" (v)            // %4
905
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
906
        );
907
        fence_after(order);
908
        return original;
909
    }
910

    
911
    static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
912
    {
913
        fence_before(order);
914
        storage_type original, result;
915
        uint32_t tmp;
916
        __asm__ __volatile__
917
        (
918
            BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
919
            "1:\n"
920
            "ldrexd  %1, %H1, [%3]\n"               // original = *(&storage)
921
            "eor     %2, %1, %4\n"                  // result = original ^ value
922
            "eor     %H2, %H1, %H4\n"
923
            "strexd  %0, %2, %H2, [%3]\n"           // *(&storage) = result, tmp = store failed
924
            "teq     %0, #0\n"                      // flags = tmp==0
925
            "bne     1b\n"                          // if (!flags.equal) goto retry
926
            BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
927
            : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
928
              "=&r" (original),  // %1
929
              "=&r" (result)     // %2
930
            : "r" (&storage),    // %3
931
              "r" (v)            // %4
932
            : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
933
        );
934
        fence_after(order);
935
        return original;
936
    }
937

    
938
    static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
939
    {
940
        return !!exchange(storage, (storage_type)1, order);
941
    }
942

    
943
    static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
944
    {
945
        store(storage, 0, order);
946
    }
947

    
948
    static BOOST_FORCEINLINE bool is_lock_free(storage_type const volatile&) BOOST_NOEXCEPT
949
    {
950
        return true;
951
    }
952
};
953

    
954
#endif // defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXD_STREXD)
955

    
956

    
957
BOOST_FORCEINLINE void thread_fence(memory_order order) BOOST_NOEXCEPT
958
{
959
    if (order != memory_order_relaxed)
960
        gcc_arm_operations_base::hardware_full_fence();
961
}
962

    
963
BOOST_FORCEINLINE void signal_fence(memory_order order) BOOST_NOEXCEPT
964
{
965
    if (order != memory_order_relaxed)
966
        __asm__ __volatile__ ("" ::: "memory");
967
}
968

    
969
} // namespace detail
970
} // namespace atomics
971
} // namespace boost
972

    
973
#endif // BOOST_ATOMIC_DETAIL_OPS_GCC_ARM_HPP_INCLUDED_