mark.c 75.4 KB
Newer Older
eg's avatar
eg committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1995 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 2000 by Hewlett-Packard Company.  All rights reserved.
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose,  provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 *
 */

17 18 19 20 21 22 23 24 25 26 27
#if defined(__MINGW32__) && !defined(__MINGW_EXCPT_DEFINE_PSDK) \
    && defined(__i386__) /* cannot use macros from gcconfig.h */
  /* Otherwise EXCEPTION_REGISTRATION type declaration from winnt.h     */
  /* might be used.  That declaration has "handler" callback with NTAPI */
  /* attribute.  The proper type (with "handler" field compatible with  */
  /* GC mark_ex_handler) is declared in excpt.h.  The given macro is    */
  /* defined before any system header include.                          */
# define __MINGW_EXCPT_DEFINE_PSDK 1
#endif

#include "private/gc_pmark.h"
28

29
#include <stdio.h>
eg's avatar
eg committed
30

eg's avatar
eg committed
31 32 33 34
#if defined(MSWIN32) && defined(__GNUC__)
# include <excpt.h>
#endif

35 36 37 38 39 40 41 42 43 44 45
/* Make arguments appear live to compiler.  Put here to minimize the    */
/* risk of inlining.  Used to minimize junk left in registers.          */
GC_ATTR_NOINLINE
void GC_noop6(word arg1 GC_ATTR_UNUSED, word arg2 GC_ATTR_UNUSED,
              word arg3 GC_ATTR_UNUSED, word arg4 GC_ATTR_UNUSED,
              word arg5 GC_ATTR_UNUSED, word arg6 GC_ATTR_UNUSED)
{
  /* Avoid GC_noop6 calls to be optimized away. */
# if defined(GC_PTHREADS) && !defined(GC_WIN32_THREADS) \
     || defined(PARALLEL_MARK)
    AO_compiler_barrier(); /* to serve as a special side-effect */
46
# else
47
    GC_noop1(0);
48
# endif
49
}
eg's avatar
eg committed
50 51

/* Single argument version, robust against whole program analysis. */
52
volatile word GC_noop_sink;
53
GC_API void GC_CALL GC_noop1(word x)
eg's avatar
eg committed
54
{
55
    GC_noop_sink = x;
eg's avatar
eg committed
56 57 58 59
}

/* mark_proc GC_mark_procs[MAX_MARK_PROCS] = {0} -- declared in gc_priv.h */

60
GC_INNER unsigned GC_n_mark_procs = GC_RESERVED_MARK_PROCS;
eg's avatar
eg committed
61

62 63 64 65 66
/* Initialize GC_obj_kinds properly and standard free lists properly.   */
/* This must be done statically since they may be accessed before       */
/* GC_init is called.                                                   */
/* It's done here, since we need to deal with mark descriptors.         */
GC_INNER struct obj_kind GC_obj_kinds[MAXOBJKINDS] = {
eg's avatar
eg committed
67
/* PTRFREE */ { &GC_aobjfreelist[0], 0 /* filled in dynamically */,
68 69 70 71 72 73 74
                /* 0 | */ GC_DS_LENGTH, FALSE, FALSE
                /*, */ OK_DISCLAIM_INITZ },
/* NORMAL */  { &GC_objfreelist[0], 0,
                /* 0 | */ GC_DS_LENGTH,
                                /* adjusted in GC_init for EXTRA_BYTES  */
                TRUE /* add length to descr */, TRUE
                /*, */ OK_DISCLAIM_INITZ },
eg's avatar
eg committed
75
/* UNCOLLECTABLE */
76 77 78 79 80 81 82
              { &GC_uobjfreelist[0], 0,
                /* 0 | */ GC_DS_LENGTH, TRUE /* add length to descr */, TRUE
                /*, */ OK_DISCLAIM_INITZ },
# ifdef GC_ATOMIC_UNCOLLECTABLE
              { &GC_auobjfreelist[0], 0,
                /* 0 | */ GC_DS_LENGTH, FALSE /* add length to descr */, FALSE
                /*, */ OK_DISCLAIM_INITZ },
eg's avatar
eg committed
83 84
# endif
# ifdef STUBBORN_ALLOC
85 86 87
              { (void **)&GC_sobjfreelist[0], 0,
                /* 0 | */ GC_DS_LENGTH, TRUE /* add length to descr */, TRUE
                /*, */ OK_DISCLAIM_INITZ },
eg's avatar
eg committed
88 89 90
# endif
};

91 92
# ifdef STUBBORN_ALLOC
#   define GC_N_KINDS_INITIAL_VALUE (STUBBORN+1)
eg's avatar
eg committed
93
# else
94
#   define GC_N_KINDS_INITIAL_VALUE STUBBORN
eg's avatar
eg committed
95 96
# endif

97
GC_INNER unsigned GC_n_kinds = GC_N_KINDS_INITIAL_VALUE;
eg's avatar
eg committed
98 99 100

# ifndef INITIAL_MARK_STACK_SIZE
#   define INITIAL_MARK_STACK_SIZE (1*HBLKSIZE)
101 102 103 104 105 106
                /* INITIAL_MARK_STACK_SIZE * sizeof(mse) should be a    */
                /* multiple of HBLKSIZE.                                */
                /* The incremental collector actually likes a larger    */
                /* size, since it wants to push all marked dirty        */
                /* objects before marking anything new.  Currently we   */
                /* let it grow dynamically.                             */
eg's avatar
eg committed
107 108
# endif

109 110 111
STATIC word GC_n_rescuing_pages = 0;
                                /* Number of dirty pages we marked from */
                                /* excludes ptrfree pages, etc.         */
eg's avatar
eg committed
112

113
GC_INNER size_t GC_mark_stack_size = 0;
eg's avatar
eg committed
114

115
#ifdef PARALLEL_MARK
116 117 118 119 120
  STATIC volatile AO_t GC_first_nonempty = 0;
        /* Lowest entry on mark stack   */
        /* that may be nonempty.        */
        /* Updated only by initiating   */
        /* thread.                      */
eg's avatar
eg committed
121 122
#endif

123
GC_INNER mark_state_t GC_mark_state = MS_NONE;
eg's avatar
eg committed
124

125
GC_INNER GC_bool GC_mark_stack_too_small = FALSE;
eg's avatar
eg committed
126

127
static struct hblk * scan_ptr;
eg's avatar
eg committed
128

129 130
STATIC GC_bool GC_objects_are_marked = FALSE;
                /* Are there collectible marked objects in the heap?    */
eg's avatar
eg committed
131

132 133 134 135
/* Is a collection in progress?  Note that this can return true in the  */
/* nonincremental case, if a collection has been abandoned and the      */
/* mark state is now MS_INVALID.                                        */
GC_INNER GC_bool GC_collection_in_progress(void)
eg's avatar
eg committed
136 137 138 139 140
{
    return(GC_mark_state != MS_NONE);
}

/* clear all mark bits in the header */
141
GC_INNER void GC_clear_hdr_marks(hdr *hhdr)
eg's avatar
eg committed
142
{
143
    size_t last_bit = FINAL_MARK_BIT(hhdr -> hb_sz);
144 145
    BZERO(hhdr -> hb_marks, sizeof(hhdr->hb_marks));
    set_mark_bit_from_hdr(hhdr, last_bit);
eg's avatar
eg committed
146
    hhdr -> hb_n_marks = 0;
eg's avatar
eg committed
147 148
}

149 150
/* Set all mark bits in the header.  Used for uncollectible blocks. */
GC_INNER void GC_set_hdr_marks(hdr *hhdr)
eg's avatar
eg committed
151
{
eg's avatar
eg committed
152 153
    unsigned i;
    size_t sz = hhdr -> hb_sz;
154
    unsigned n_marks = (unsigned)FINAL_MARK_BIT(sz);
eg's avatar
eg committed
155

eg's avatar
eg committed
156
#   ifdef USE_MARK_BYTES
157
      for (i = 0; i <= n_marks; i += (unsigned)MARK_BIT_OFFSET(sz)) {
158
        hhdr -> hb_marks[i] = 1;
eg's avatar
eg committed
159 160 161
      }
#   else
      for (i = 0; i < divWORDSZ(n_marks + WORDSZ); ++i) {
162
        hhdr -> hb_marks[i] = ONES;
eg's avatar
eg committed
163 164 165
      }
#   endif
#   ifdef MARK_BIT_PER_OBJ
166
      hhdr -> hb_n_marks = n_marks;
eg's avatar
eg committed
167 168 169
#   else
      hhdr -> hb_n_marks = HBLK_OBJS(sz);
#   endif
eg's avatar
eg committed
170 171 172 173 174
}

/*
 * Clear all mark bits associated with block h.
 */
175
static void clear_marks_for_block(struct hblk *h, word dummy GC_ATTR_UNUSED)
eg's avatar
eg committed
176 177
{
    register hdr * hhdr = HDR(h);
178

eg's avatar
eg committed
179
    if (IS_UNCOLLECTABLE(hhdr -> hb_obj_kind)) return;
180 181 182
        /* Mark bit for these is cleared only once the object is        */
        /* explicitly deallocated.  This either frees the block, or     */
        /* the bit is cleared once the object is on the free list.      */
eg's avatar
eg committed
183 184 185 186
    GC_clear_hdr_marks(hhdr);
}

/* Slow but general routines for setting/clearing/asking about mark bits */
187
GC_API void GC_CALL GC_set_mark_bit(const void *p)
eg's avatar
eg committed
188
{
eg's avatar
eg committed
189 190
    struct hblk *h = HBLKPTR(p);
    hdr * hhdr = HDR(h);
191 192
    word bit_no = MARK_BIT_NO((ptr_t)p - (ptr_t)h, hhdr -> hb_sz);

eg's avatar
eg committed
193 194 195 196
    if (!mark_bit_from_hdr(hhdr, bit_no)) {
      set_mark_bit_from_hdr(hhdr, bit_no);
      ++hhdr -> hb_n_marks;
    }
eg's avatar
eg committed
197 198
}

199
GC_API void GC_CALL GC_clear_mark_bit(const void *p)
eg's avatar
eg committed
200
{
eg's avatar
eg committed
201 202
    struct hblk *h = HBLKPTR(p);
    hdr * hhdr = HDR(h);
203 204
    word bit_no = MARK_BIT_NO((ptr_t)p - (ptr_t)h, hhdr -> hb_sz);

eg's avatar
eg committed
205
    if (mark_bit_from_hdr(hhdr, bit_no)) {
206 207 208
      size_t n_marks = hhdr -> hb_n_marks;

      GC_ASSERT(n_marks != 0);
eg's avatar
eg committed
209
      clear_mark_bit_from_hdr(hhdr, bit_no);
210
      n_marks--;
eg's avatar
eg committed
211
#     ifdef PARALLEL_MARK
212
        if (n_marks != 0 || !GC_parallel)
213 214 215 216
          hhdr -> hb_n_marks = n_marks;
        /* Don't decrement to zero.  The counts are approximate due to  */
        /* concurrency issues, but we need to ensure that a count of    */
        /* zero implies an empty block.                                 */
eg's avatar
eg committed
217
#     else
218
          hhdr -> hb_n_marks = n_marks;
eg's avatar
eg committed
219 220
#     endif
    }
eg's avatar
eg committed
221 222
}

223
GC_API int GC_CALL GC_is_marked(const void *p)
eg's avatar
eg committed
224
{
eg's avatar
eg committed
225 226
    struct hblk *h = HBLKPTR(p);
    hdr * hhdr = HDR(h);
227
    word bit_no = MARK_BIT_NO((ptr_t)p - (ptr_t)h, hhdr -> hb_sz);
eg's avatar
eg committed
228

229 230
    return (int)mark_bit_from_hdr(hhdr, bit_no); /* 0 or 1 */
}
eg's avatar
eg committed
231 232 233 234 235 236

/*
 * Clear mark bits in all allocated heap blocks.  This invalidates
 * the marker invariant, and sets GC_mark_state to reflect this.
 * (This implicitly starts marking to reestablish the invariant.)
 */
237
GC_INNER void GC_clear_marks(void)
eg's avatar
eg committed
238 239 240 241 242 243 244
{
    GC_apply_to_all_blocks(clear_marks_for_block, (word)0);
    GC_objects_are_marked = FALSE;
    GC_mark_state = MS_INVALID;
    scan_ptr = 0;
}

245
#ifdef CHECKSUMS
246
  void GC_check_dirty(void);
247 248
#endif

249 250 251
/* Initiate a garbage collection.  Initiates a full collection if the   */
/* mark state is invalid.                                               */
GC_INNER void GC_initiate_gc(void)
eg's avatar
eg committed
252
{
253 254 255 256 257
#   ifndef GC_DISABLE_INCREMENTAL
        if (GC_incremental) {
            GC_read_dirty();
        }
#   endif
eg's avatar
eg committed
258
#   ifdef STUBBORN_ALLOC
259
        GC_read_changed();
eg's avatar
eg committed
260 261
#   endif
#   ifdef CHECKSUMS
262
        if (GC_incremental) GC_check_dirty();
eg's avatar
eg committed
263 264 265 266 267
#   endif
    GC_n_rescuing_pages = 0;
    if (GC_mark_state == MS_NONE) {
        GC_mark_state = MS_PUSH_RESCUERS;
    } else if (GC_mark_state != MS_INVALID) {
268 269 270
        ABORT("Unexpected state");
    } /* else this is really a full collection, and mark        */
      /* bits are invalid.                                      */
eg's avatar
eg committed
271 272 273
    scan_ptr = 0;
}

274 275 276
#ifdef PARALLEL_MARK
    STATIC void GC_do_parallel_mark(void); /* initiate parallel marking. */
#endif /* PARALLEL_MARK */
eg's avatar
eg committed
277

278 279 280 281 282 283 284 285 286 287 288
#ifdef GC_DISABLE_INCREMENTAL
# define GC_push_next_marked_dirty(h) GC_push_next_marked(h)
#else
  STATIC struct hblk * GC_push_next_marked_dirty(struct hblk *h);
                /* Invoke GC_push_marked on next dirty block above h.   */
                /* Return a pointer just past the end of this block.    */
#endif /* !GC_DISABLE_INCREMENTAL */
STATIC struct hblk * GC_push_next_marked(struct hblk *h);
                /* Ditto, but also mark from clean pages.       */
STATIC struct hblk * GC_push_next_marked_uncollectable(struct hblk *h);
                /* Ditto, but mark only from uncollectible pages.       */
eg's avatar
eg committed
289

290
static void alloc_mark_stack(size_t);
eg's avatar
eg committed
291

292 293 294 295 296 297 298 299
/* Perform a small amount of marking.                   */
/* We try to touch roughly a page of memory.            */
/* Return TRUE if we just finished a mark phase.        */
/* Cold_gc_frame is an address inside a GC frame that   */
/* remains valid until all marking is complete.         */
/* A zero value indicates that it's OK to miss some     */
/* register values.                                     */
/* We hold the allocation lock.  In the case of         */
eg's avatar
eg committed
300 301
/* incremental collection, the world may not be stopped.*/
#ifdef WRAP_MARK_SOME
302 303 304 305
  /* For win32, this is called after we establish a structured  */
  /* exception handler, in case Windows unmaps one of our root  */
  /* segments.  See below.  In either case, we acquire the      */
  /* allocator lock long before we get here.                    */
306
  STATIC GC_bool GC_mark_some_inner(ptr_t cold_gc_frame)
eg's avatar
eg committed
307
#else
308
  GC_INNER GC_bool GC_mark_some(ptr_t cold_gc_frame)
eg's avatar
eg committed
309
#endif
eg's avatar
eg committed
310 311
{
    switch(GC_mark_state) {
312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424
        case MS_NONE:
            break;

        case MS_PUSH_RESCUERS:
            if ((word)GC_mark_stack_top
                >= (word)(GC_mark_stack_limit - INITIAL_MARK_STACK_SIZE/2)) {
                /* Go ahead and mark, even though that might cause us to */
                /* see more marked dirty objects later on.  Avoid this   */
                /* in the future.                                        */
                GC_mark_stack_too_small = TRUE;
                MARK_FROM_MARK_STACK();
                break;
            } else {
                scan_ptr = GC_push_next_marked_dirty(scan_ptr);
                if (scan_ptr == 0) {
                    GC_COND_LOG_PRINTF("Marked from %lu dirty pages\n",
                                       (unsigned long)GC_n_rescuing_pages);
                    GC_push_roots(FALSE, cold_gc_frame);
                    GC_objects_are_marked = TRUE;
                    if (GC_mark_state != MS_INVALID) {
                        GC_mark_state = MS_ROOTS_PUSHED;
                    }
                }
            }
            break;

        case MS_PUSH_UNCOLLECTABLE:
            if ((word)GC_mark_stack_top
                >= (word)(GC_mark_stack + GC_mark_stack_size/4)) {
#               ifdef PARALLEL_MARK
                  /* Avoid this, since we don't parallelize the marker  */
                  /* here.                                              */
                  if (GC_parallel) GC_mark_stack_too_small = TRUE;
#               endif
                MARK_FROM_MARK_STACK();
                break;
            } else {
                scan_ptr = GC_push_next_marked_uncollectable(scan_ptr);
                if (scan_ptr == 0) {
                    GC_push_roots(TRUE, cold_gc_frame);
                    GC_objects_are_marked = TRUE;
                    if (GC_mark_state != MS_INVALID) {
                        GC_mark_state = MS_ROOTS_PUSHED;
                    }
                }
            }
            break;

        case MS_ROOTS_PUSHED:
#           ifdef PARALLEL_MARK
              /* In the incremental GC case, this currently doesn't     */
              /* quite do the right thing, since it runs to             */
              /* completion.  On the other hand, starting a             */
              /* parallel marker is expensive, so perhaps it is         */
              /* the right thing?                                       */
              /* Eventually, incremental marking should run             */
              /* asynchronously in multiple threads, without grabbing   */
              /* the allocation lock.                                   */
                if (GC_parallel) {
                  GC_do_parallel_mark();
                  GC_ASSERT((word)GC_mark_stack_top < (word)GC_first_nonempty);
                  GC_mark_stack_top = GC_mark_stack - 1;
                  if (GC_mark_stack_too_small) {
                    alloc_mark_stack(2*GC_mark_stack_size);
                  }
                  if (GC_mark_state == MS_ROOTS_PUSHED) {
                    GC_mark_state = MS_NONE;
                    return(TRUE);
                  }
                  break;
                }
#           endif
            if ((word)GC_mark_stack_top >= (word)GC_mark_stack) {
                MARK_FROM_MARK_STACK();
                break;
            } else {
                GC_mark_state = MS_NONE;
                if (GC_mark_stack_too_small) {
                    alloc_mark_stack(2*GC_mark_stack_size);
                }
                return(TRUE);
            }

        case MS_INVALID:
        case MS_PARTIALLY_INVALID:
            if (!GC_objects_are_marked) {
                GC_mark_state = MS_PUSH_UNCOLLECTABLE;
                break;
            }
            if ((word)GC_mark_stack_top >= (word)GC_mark_stack) {
                MARK_FROM_MARK_STACK();
                break;
            }
            if (scan_ptr == 0 && GC_mark_state == MS_INVALID) {
                /* About to start a heap scan for marked objects. */
                /* Mark stack is empty.  OK to reallocate.        */
                if (GC_mark_stack_too_small) {
                    alloc_mark_stack(2*GC_mark_stack_size);
                }
                GC_mark_state = MS_PARTIALLY_INVALID;
            }
            scan_ptr = GC_push_next_marked(scan_ptr);
            if (scan_ptr == 0 && GC_mark_state == MS_PARTIALLY_INVALID) {
                GC_push_roots(TRUE, cold_gc_frame);
                GC_objects_are_marked = TRUE;
                if (GC_mark_state != MS_INVALID) {
                    GC_mark_state = MS_ROOTS_PUSHED;
                }
            }
            break;

        default:
            ABORT("GC_mark_some: bad state");
eg's avatar
eg committed
425
    }
426
    return(FALSE);
eg's avatar
eg committed
427 428
}

429
#ifdef WRAP_MARK_SOME
eg's avatar
eg committed
430

431
# if (defined(MSWIN32) || defined(MSWINCE)) && defined(__GNUC__)
eg's avatar
eg committed
432

eg's avatar
eg committed
433 434 435 436
    typedef struct {
      EXCEPTION_REGISTRATION ex_reg;
      void *alt_path;
    } ext_ex_regn;
eg's avatar
eg committed
437

eg's avatar
eg committed
438
    static EXCEPTION_DISPOSITION mark_ex_handler(
439
        struct _EXCEPTION_RECORD *ex_rec,
eg's avatar
eg committed
440 441
        void *est_frame,
        struct _CONTEXT *context,
442
        void *disp_ctxt GC_ATTR_UNUSED)
eg's avatar
eg committed
443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462
    {
        if (ex_rec->ExceptionCode == STATUS_ACCESS_VIOLATION) {
          ext_ex_regn *xer = (ext_ex_regn *)est_frame;

          /* Unwind from the inner function assuming the standard */
          /* function prologue.                                   */
          /* Assumes code has not been compiled with              */
          /* -fomit-frame-pointer.                                */
          context->Esp = context->Ebp;
          context->Ebp = *((DWORD *)context->Esp);
          context->Esp = context->Esp - 8;

          /* Resume execution at the "real" handler within the    */
          /* wrapper function.                                    */
          context->Eip = (DWORD )(xer->alt_path);

          return ExceptionContinueExecution;

        } else {
            return ExceptionContinueSearch;
eg's avatar
eg committed
463 464
        }
    }
465
# endif /* __GNUC__ && MSWIN32 */
eg's avatar
eg committed
466

467 468 469 470
#if defined(GC_WIN32_THREADS) && !defined(GC_PTHREADS)
  GC_INNER GC_bool GC_started_thread_while_stopped(void);
  /* In win32_threads.c.  Did we invalidate mark phase with an  */
  /* unexpected thread start?                                   */
471
#endif
eg's avatar
eg committed
472

473
  GC_INNER GC_bool GC_mark_some(ptr_t cold_gc_frame)
eg's avatar
eg committed
474 475 476
  {
      GC_bool ret_val;

477
#   if defined(MSWIN32) || defined(MSWINCE)
eg's avatar
eg committed
478 479 480 481 482 483 484 485
#    ifndef __GNUC__
      /* Windows 98 appears to asynchronously create and remove  */
      /* writable memory mappings, for reasons we haven't yet    */
      /* understood.  Since we look for writable regions to      */
      /* determine the root set, we may try to mark from an      */
      /* address range that disappeared since we started the     */
      /* collection.  Thus we have to recover from faults here.  */
      /* This code does not appear to be necessary for Windows   */
486
      /* 95/NT/2000+. Note that this code should never generate  */
eg's avatar
eg committed
487
      /* an incremental GC write fault.                          */
488 489 490 491 492 493
      /* This code seems to be necessary for WinCE (at least in  */
      /* the case we'd decide to add MEM_PRIVATE sections to     */
      /* data roots in GC_register_dynamic_libraries()).         */
      /* It's conceivable that this is the same issue with       */
      /* terminating threads that we see with Linux and          */
      /* USE_PROC_FOR_LIBRARIES.                                 */
eg's avatar
eg committed
494 495

      __try {
496 497 498
          ret_val = GC_mark_some_inner(cold_gc_frame);
      } __except (GetExceptionCode() == EXCEPTION_ACCESS_VIOLATION ?
                EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH) {
499
          goto handle_ex;
500
      }
501 502 503 504 505 506 507
#     if defined(GC_WIN32_THREADS) && !defined(GC_PTHREADS)
        /* With DllMain-based thread tracking, a thread may have        */
        /* started while we were marking.  This is logically equivalent */
        /* to the exception case; our results are invalid and we have   */
        /* to start over.  This cannot be prevented since we can't      */
        /* block in DllMain.                                            */
        if (GC_started_thread_while_stopped()) goto handle_ex;
508 509 510
#     endif
     rm_handler:
      return ret_val;
eg's avatar
eg committed
511 512 513 514 515 516 517 518 519

#    else /* __GNUC__ */

      /* Manually install an exception handler since GCC does    */
      /* not yet support Structured Exception Handling (SEH) on  */
      /* Win32.                                                  */

      ext_ex_regn er;

520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536
#     if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 7) \
         || __clang_major__ > 3 \
         || (__clang_major__ == 3 && __clang_minor__ >= 3)
#       pragma GCC diagnostic push
        /* Suppress "taking the address of label is non-standard" warning. */
#       if defined(__clang__) || __GNUC__ > 6 \
           || (__GNUC__ == 6 && __GNUC_MINOR__ >= 4)
#         pragma GCC diagnostic ignored "-Wpedantic"
#       else
          /* GCC before ~4.8 does not accept "-Wpedantic" quietly.  */
#         pragma GCC diagnostic ignored "-pedantic"
#       endif
        er.alt_path = &&handle_ex;
#       pragma GCC diagnostic pop
#     else /* pragma diagnostic is not supported */
        er.alt_path = &&handle_ex;
#     endif
eg's avatar
eg committed
537
      er.ex_reg.handler = mark_ex_handler;
538 539
      __asm__ __volatile__ ("movl %%fs:0, %0" : "=r" (er.ex_reg.prev));
      __asm__ __volatile__ ("movl %0, %%fs:0" : : "r" (&er));
540 541 542 543 544
      ret_val = GC_mark_some_inner(cold_gc_frame);
      /* Prevent GCC from considering the following code unreachable */
      /* and thus eliminating it.                                    */
        if (er.alt_path == 0)
          goto handle_ex;
545 546 547 548
#     if defined(GC_WIN32_THREADS) && !defined(GC_PTHREADS)
        if (GC_started_thread_while_stopped())
          goto handle_ex;
#     endif
549 550
    rm_handler:
      /* Uninstall the exception handler */
551
      __asm__ __volatile__ ("mov %0, %%fs:0" : : "r" (er.ex_reg.prev));
552
      return ret_val;
eg's avatar
eg committed
553 554 555

#    endif /* __GNUC__ */
#   else /* !MSWIN32 */
556 557 558 559 560 561 562 563 564
      /* Here we are handling the case in which /proc is used for root  */
      /* finding, and we have threads.  We may find a stack for a       */
      /* thread that is in the process of exiting, and disappears       */
      /* while we are marking it.  This seems extremely difficult to    */
      /* avoid otherwise.                                               */
      if (GC_incremental) {
        WARN("Incremental GC incompatible with /proc roots\n", 0);
        /* I'm not sure if this could still work ...    */
      }
eg's avatar
eg committed
565 566
      GC_setup_temporary_fault_handler();
      if(SETJMP(GC_jmp_buf) != 0) goto handle_ex;
567 568 569 570
      ret_val = GC_mark_some_inner(cold_gc_frame);
    rm_handler:
      GC_reset_fault_handler();
      return ret_val;
571

eg's avatar
eg committed
572 573 574
#   endif /* !MSWIN32 */

handle_ex:
575
    /* Exception handler starts here for all cases. */
576 577 578 579 580 581 582 583 584
      {
        static word warned_gc_no;

        /* Warn about it at most once per collection. */
        if (warned_gc_no != GC_gc_no) {
          warned_gc_no = GC_gc_no;
          WARN("Caught ACCESS_VIOLATION in marker;"
               " memory mapping disappeared\n", 0);
        }
eg's avatar
eg committed
585
      }
586 587 588 589 590 591 592 593
      /* We have bad roots on the stack.  Discard mark stack.   */
      /* Rescan from marked objects.  Redetermine roots.        */
#     ifdef REGISTER_LIBRARIES_EARLY
        START_WORLD();
        GC_cond_register_dynamic_libraries();
        STOP_WORLD();
#     endif
      GC_invalidate_mark_state();
594
      scan_ptr = 0;
eg's avatar
eg committed
595

596
      ret_val = FALSE;
597
      goto rm_handler;  /* Back to platform-specific code. */
eg's avatar
eg committed
598 599 600
  }
#endif /* WRAP_MARK_SOME */

601
GC_INNER void GC_invalidate_mark_state(void)
eg's avatar
eg committed
602 603 604 605 606
{
    GC_mark_state = MS_INVALID;
    GC_mark_stack_top = GC_mark_stack-1;
}

607
GC_INNER mse * GC_signal_mark_stack_overflow(mse *msp)
eg's avatar
eg committed
608 609
{
    GC_mark_state = MS_INVALID;
610 611 612 613 614 615 616 617 618 619 620
#   ifdef PARALLEL_MARK
      /* We are using a local_mark_stack in parallel mode, so   */
      /* do not signal the global mark stack to be resized.     */
      /* That will be done if required in GC_return_mark_stack. */
      if (!GC_parallel)
        GC_mark_stack_too_small = TRUE;
#   else
      GC_mark_stack_too_small = TRUE;
#   endif
    GC_COND_LOG_PRINTF("Mark stack overflow; current size = %lu entries\n",
                       (unsigned long)GC_mark_stack_size);
eg's avatar
eg committed
621 622 623 624 625
    return(msp - GC_MARK_STACK_DISCARDS);
}

/*
 * Mark objects pointed to by the regions described by
eg's avatar
eg committed
626
 * mark stack entries between mark_stack and mark_stack_top,
eg's avatar
eg committed
627 628 629 630 631 632 633 634 635
 * inclusive.  Assumes the upper limit of a mark stack entry
 * is never 0.  A mark stack entry never has size 0.
 * We try to traverse on the order of a hblk of memory before we return.
 * Caller is responsible for calling this until the mark stack is empty.
 * Note that this is the most performance critical routine in the
 * collector.  Hence it contains all sorts of ugly hacks to speed
 * things up.  In particular, we avoid procedure calls on the common
 * path, we take advantage of peculiarities of the mark descriptor
 * encoding, we optionally maintain a cache for the block address to
636
 * header mapping, we prefetch when an object is "grayed", etc.
eg's avatar
eg committed
637
 */
638 639 640
GC_ATTR_NO_SANITIZE_ADDR GC_ATTR_NO_SANITIZE_MEMORY
GC_INNER mse * GC_mark_from(mse *mark_stack_top, mse *mark_stack,
                            mse *mark_stack_limit)
eg's avatar
eg committed
641
{
642 643 644 645
  signed_word credit = HBLKSIZE;  /* Remaining credit for marking work  */
  ptr_t current_p;      /* Pointer to current candidate ptr.            */
  word current;         /* Candidate pointer.                           */
  ptr_t limit = 0;      /* (Incl) limit of current candidate range.     */
eg's avatar
eg committed
646 647 648
  word descr;
  ptr_t greatest_ha = GC_greatest_plausible_heap_addr;
  ptr_t least_ha = GC_least_plausible_heap_addr;
eg's avatar
eg committed
649 650
  DECLARE_HDR_CACHE;

651
# define SPLIT_RANGE_WORDS 128  /* Must be power of 2.          */
eg's avatar
eg committed
652 653 654 655

  GC_objects_are_marked = TRUE;
  INIT_HDR_CACHE;
# ifdef OS2 /* Use untweaked version to circumvent compiler problem */
656
    while ((word)mark_stack_top >= (word)mark_stack && credit >= 0)
eg's avatar
eg committed
657
# else
658
    while ((((ptr_t)mark_stack_top - (ptr_t)mark_stack) | credit) >= 0)
eg's avatar
eg committed
659
# endif
660
  {
eg's avatar
eg committed
661
    current_p = mark_stack_top -> mse_start;
662
    descr = mark_stack_top -> mse_descr.w;
eg's avatar
eg committed
663
  retry:
664 665 666 667 668
    /* current_p and descr describe the current object.         */
    /* *mark_stack_top is vacant.                               */
    /* The following is 0 only for small objects described by a simple  */
    /* length descriptor.  For many applications this is the common     */
    /* case, so we try to detect it quickly.                            */
eg's avatar
eg committed
669 670
    if (descr & ((~(WORDS_TO_BYTES(SPLIT_RANGE_WORDS) - 1)) | GC_DS_TAGS)) {
      word tag = descr & GC_DS_TAGS;
671 672

      GC_STATIC_ASSERT(GC_DS_TAGS == 0x3);
eg's avatar
eg committed
673 674
      switch(tag) {
        case GC_DS_LENGTH:
675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708
          /* Large length.                                              */
          /* Process part of the range to avoid pushing too much on the */
          /* stack.                                                     */
          GC_ASSERT(descr < (word)GC_greatest_plausible_heap_addr
                            - (word)GC_least_plausible_heap_addr
                || (word)(current_p + descr)
                            <= (word)GC_least_plausible_heap_addr
                || (word)current_p >= (word)GC_greatest_plausible_heap_addr);
#         ifdef PARALLEL_MARK
#           define SHARE_BYTES 2048
            if (descr > SHARE_BYTES && GC_parallel
                && (word)mark_stack_top < (word)(mark_stack_limit - 1)) {
              word new_size = (descr/2) & ~(word)(sizeof(word)-1);

              mark_stack_top -> mse_start = current_p;
              mark_stack_top -> mse_descr.w = new_size + sizeof(word);
                                        /* makes sure we handle         */
                                        /* misaligned pointers.         */
              mark_stack_top++;
#             ifdef ENABLE_TRACE
                if ((word)GC_trace_addr >= (word)current_p
                    && (word)GC_trace_addr < (word)(current_p + descr)) {
                  GC_log_printf("GC #%u: large section; start %p, len %lu,"
                                " splitting (parallel) at %p\n",
                                (unsigned)GC_gc_no, (void *)current_p,
                                (unsigned long)descr,
                                (void *)(current_p + new_size));
                }
#             endif
              current_p += new_size;
              descr -= new_size;
              goto retry;
            }
#         endif /* PARALLEL_MARK */
eg's avatar
eg committed
709
          mark_stack_top -> mse_start =
710 711 712 713 714 715 716 717 718 719 720 721 722 723
                limit = current_p + WORDS_TO_BYTES(SPLIT_RANGE_WORDS-1);
          mark_stack_top -> mse_descr.w =
                                descr - WORDS_TO_BYTES(SPLIT_RANGE_WORDS-1);
#         ifdef ENABLE_TRACE
            if ((word)GC_trace_addr >= (word)current_p
                && (word)GC_trace_addr < (word)(current_p + descr)) {
              GC_log_printf("GC #%u: large section; start %p, len %lu,"
                            " splitting at %p\n",
                            (unsigned)GC_gc_no, (void *)current_p,
                            (unsigned long)descr, (void *)limit);
            }
#         endif
          /* Make sure that pointers overlapping the two ranges are     */
          /* considered.                                                */
eg's avatar
eg committed
724
          limit += sizeof(word) - ALIGNMENT;
eg's avatar
eg committed
725 726 727
          break;
        case GC_DS_BITMAP:
          mark_stack_top--;
728 729 730 731 732 733 734 735 736
#         ifdef ENABLE_TRACE
            if ((word)GC_trace_addr >= (word)current_p
                && (word)GC_trace_addr < (word)(current_p
                                                + WORDS_TO_BYTES(WORDSZ-2))) {
              GC_log_printf("GC #%u: tracing from %p bitmap descr %lu\n",
                            (unsigned)GC_gc_no, (void *)current_p,
                            (unsigned long)descr);
            }
#         endif /* ENABLE_TRACE */
eg's avatar
eg committed
737 738 739
          descr &= ~GC_DS_TAGS;
          credit -= WORDS_TO_BYTES(WORDSZ/2); /* guess */
          while (descr != 0) {
740
            if ((descr & SIGNB) != 0) {
eg's avatar
eg committed
741
              current = *(word *)current_p;
742 743 744
              FIXUP_POINTER(current);
              if (current >= (word)least_ha && current < (word)greatest_ha) {
                PREFETCH((ptr_t)current);
eg's avatar
eg committed
745
#               ifdef ENABLE_TRACE
746 747 748 749 750
                  if (GC_trace_addr == current_p) {
                    GC_log_printf("GC #%u: considering(3) %p -> %p\n",
                                  (unsigned)GC_gc_no, (void *)current_p,
                                  (void *)current);
                  }
eg's avatar
eg committed
751 752
#               endif /* ENABLE_TRACE */
                PUSH_CONTENTS((ptr_t)current, mark_stack_top,
753 754
                              mark_stack_limit, current_p);
              }
eg's avatar
eg committed
755
            }
756 757
            descr <<= 1;
            current_p += sizeof(word);
eg's avatar
eg committed
758 759 760 761
          }
          continue;
        case GC_DS_PROC:
          mark_stack_top--;
762 763 764 765 766 767 768 769 770
#         ifdef ENABLE_TRACE
            if ((word)GC_trace_addr >= (word)current_p
                && GC_base(current_p) != 0
                && GC_base(current_p) == GC_base(GC_trace_addr)) {
              GC_log_printf("GC #%u: tracing from %p, proc descr %lu\n",
                            (unsigned)GC_gc_no, (void *)current_p,
                            (unsigned long)descr);
            }
#         endif /* ENABLE_TRACE */
eg's avatar
eg committed
771
          credit -= GC_PROC_BYTES;
772 773
          mark_stack_top = (*PROC(descr))((word *)current_p, mark_stack_top,
                                          mark_stack_limit, ENV(descr));
eg's avatar
eg committed
774 775
          continue;
        case GC_DS_PER_OBJECT:
776 777
          if ((signed_word)descr >= 0) {
            /* Descriptor is in the object.     */
eg's avatar
eg committed
778
            descr = *(word *)(current_p + descr - GC_DS_PER_OBJECT);
779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794
          } else {
            /* Descriptor is in type descriptor pointed to by first     */
            /* word in object.                                          */
            ptr_t type_descr = *(ptr_t *)current_p;
            /* type_descr is either a valid pointer to the descriptor   */
            /* structure, or this object was on a free list.            */
            /* If it was anything but the last object on the free list, */
            /* we will misinterpret the next object on the free list as */
            /* the type descriptor, and get a 0 GC descriptor, which    */
            /* is ideal.  Unfortunately, we need to check for the last  */
            /* object case explicitly.                                  */
            if (0 == type_descr) {
                /* Rarely executed.     */
                mark_stack_top--;
                continue;
            }
eg's avatar
eg committed
795
            descr = *(word *)(type_descr
796 797 798 799 800 801 802 803 804
                              - ((signed_word)descr + (GC_INDIR_PER_OBJ_BIAS
                                                       - GC_DS_PER_OBJECT)));
          }
          if (0 == descr) {
              /* Can happen either because we generated a 0 descriptor  */
              /* or we saw a pointer to a free object.          */
              mark_stack_top--;
              continue;
          }
eg's avatar
eg committed
805 806 807 808
          goto retry;
      }
    } else /* Small object with length descriptor */ {
      mark_stack_top--;
809 810 811 812 813 814 815 816 817 818 819 820
#     ifndef SMALL_CONFIG
        if (descr < sizeof(word))
          continue;
#     endif
#     ifdef ENABLE_TRACE
        if ((word)GC_trace_addr >= (word)current_p
            && (word)GC_trace_addr < (word)(current_p + descr)) {
          GC_log_printf("GC #%u: small object; start %p, len %lu\n",
                        (unsigned)GC_gc_no, (void *)current_p,
                        (unsigned long)descr);
        }
#     endif
eg's avatar
eg committed
821
      limit = current_p + (word)descr;
eg's avatar
eg committed
822
    }
823
    /* The simple case in which we're scanning a range. */
eg's avatar
eg committed
824
    GC_ASSERT(!((word)current_p & (ALIGNMENT-1)));
eg's avatar
eg committed
825 826
    credit -= limit - current_p;
    limit -= sizeof(word);
eg's avatar
eg committed
827 828 829 830 831 832
    {
#     define PREF_DIST 4

#     ifndef SMALL_CONFIG
        word deferred;

833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860
        /* Try to prefetch the next pointer to be examined ASAP.        */
        /* Empirically, this also seems to help slightly without        */
        /* prefetches, at least on linux/X86.  Presumably this loop     */
        /* ends up with less register pressure, and gcc thus ends up    */
        /* generating slightly better code.  Overall gcc code quality   */
        /* for this loop is still not great.                            */
        for(;;) {
          PREFETCH(limit - PREF_DIST*CACHE_LINE_SIZE);
          GC_ASSERT((word)limit >= (word)current_p);
          deferred = *(word *)limit;
          FIXUP_POINTER(deferred);
          limit -= ALIGNMENT;
          if (deferred >= (word)least_ha && deferred < (word)greatest_ha) {
            PREFETCH((ptr_t)deferred);
            break;
          }
          if ((word)current_p > (word)limit) goto next_object;
          /* Unroll once, so we don't do too many of the prefetches     */
          /* based on limit.                                            */
          deferred = *(word *)limit;
          FIXUP_POINTER(deferred);
          limit -= ALIGNMENT;
          if (deferred >= (word)least_ha && deferred < (word)greatest_ha) {
            PREFETCH((ptr_t)deferred);
            break;
          }
          if ((word)current_p > (word)limit) goto next_object;
        }
eg's avatar
eg committed
861 862
#     endif

863 864 865 866
      while ((word)current_p <= (word)limit) {
        /* Empirically, unrolling this loop doesn't help a lot. */
        /* Since PUSH_CONTENTS expands to a lot of code,        */
        /* we don't.                                            */
eg's avatar
eg committed
867
        current = *(word *)current_p;
868
        FIXUP_POINTER(current);
eg's avatar
eg committed
869
        PREFETCH(current_p + PREF_DIST*CACHE_LINE_SIZE);
870 871 872 873
        if (current >= (word)least_ha && current < (word)greatest_ha) {
          /* Prefetch the contents of the object we just pushed.  It's  */
          /* likely we will need them soon.                             */
          PREFETCH((ptr_t)current);
eg's avatar
eg committed
874
#         ifdef ENABLE_TRACE
875 876 877 878 879
            if (GC_trace_addr == current_p) {
              GC_log_printf("GC #%u: considering(1) %p -> %p\n",
                            (unsigned)GC_gc_no, (void *)current_p,
                            (void *)current);
            }
eg's avatar
eg committed
880 881
#         endif /* ENABLE_TRACE */
          PUSH_CONTENTS((ptr_t)current, mark_stack_top,
882
                        mark_stack_limit, current_p);
eg's avatar
eg committed
883
        }
eg's avatar
eg committed
884
        current_p += ALIGNMENT;
eg's avatar
eg committed
885 886 887
      }

#     ifndef SMALL_CONFIG
888 889 890
        /* We still need to mark the entry we previously prefetched.    */
        /* We already know that it passes the preliminary pointer       */
        /* validity test.                                               */
eg's avatar
eg committed
891
#       ifdef ENABLE_TRACE
892 893 894 895 896
            if (GC_trace_addr == current_p) {
              GC_log_printf("GC #%u: considering(2) %p -> %p\n",
                            (unsigned)GC_gc_no, (void *)current_p,
                            (void *)deferred);
            }
eg's avatar
eg committed
897 898
#       endif /* ENABLE_TRACE */
        PUSH_CONTENTS((ptr_t)deferred, mark_stack_top,
899 900
                      mark_stack_limit, current_p);
        next_object:;
eg's avatar
eg committed
901 902 903 904 905 906 907 908
#     endif
    }
  }
  return mark_stack_top;
}

#ifdef PARALLEL_MARK

909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955
STATIC GC_bool GC_help_wanted = FALSE;  /* Protected by mark lock       */
STATIC unsigned GC_helper_count = 0;    /* Number of running helpers.   */
                                        /* Protected by mark lock       */
STATIC unsigned GC_active_count = 0;    /* Number of active helpers.    */
                                        /* Protected by mark lock       */
                                        /* May increase and decrease    */
                                        /* within each mark cycle.  But */
                                        /* once it returns to 0, it     */
                                        /* stays zero for the cycle.    */

GC_INNER word GC_mark_no = 0;

#ifdef LINT2
# define LOCAL_MARK_STACK_SIZE (HBLKSIZE / 8)
#else
# define LOCAL_MARK_STACK_SIZE HBLKSIZE
        /* Under normal circumstances, this is big enough to guarantee  */
        /* we don't overflow half of it in a single call to             */
        /* GC_mark_from.                                                */
#endif

/* Wait all markers to finish initialization (i.e. store        */
/* marker_[b]sp, marker_mach_threads, GC_marker_Id).            */
GC_INNER void GC_wait_for_markers_init(void)
{
  signed_word count;

  if (GC_markers_m1 == 0)
    return;

  /* Reuse marker lock and builders count to synchronize        */
  /* marker threads startup.                                    */
  GC_acquire_mark_lock();
  GC_fl_builder_count += GC_markers_m1;
  count = GC_fl_builder_count;
  GC_release_mark_lock();
  if (count != 0) {
    GC_ASSERT(count > 0);
    GC_wait_for_reclaim();
  }
}

/* Steal mark stack entries starting at mse low into mark stack local   */
/* until we either steal mse high, or we have max entries.              */
/* Return a pointer to the top of the local mark stack.                 */
/* *next is replaced by a pointer to the next unscanned mark stack      */
/* entry.                                                               */
956
STATIC mse * GC_steal_mark_stack(mse * low, mse * high, mse * local,
957
                                 unsigned max, mse **next)
eg's avatar
eg committed
958 959 960 961 962
{
    mse *p;
    mse *top = local - 1;
    unsigned i = 0;

963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986
    GC_ASSERT((word)high >= (word)(low - 1)
              && (word)(high - low + 1) <= GC_mark_stack_size);
    for (p = low; (word)p <= (word)high && i <= max; ++p) {
        word descr = (word)AO_load(&p->mse_descr.ao);
        if (descr != 0) {
            /* Must be ordered after read of descr: */
            AO_store_release_write(&p->mse_descr.ao, 0);
            /* More than one thread may get this entry, but that's only */
            /* a minor performance problem.                             */
            ++top;
            top -> mse_descr.w = descr;
            top -> mse_start = p -> mse_start;
            GC_ASSERT((descr & GC_DS_TAGS) != GC_DS_LENGTH
                      || descr < (word)GC_greatest_plausible_heap_addr
                                        - (word)GC_least_plausible_heap_addr
                      || (word)(p->mse_start + descr)
                            <= (word)GC_least_plausible_heap_addr
                      || (word)p->mse_start
                            >= (word)GC_greatest_plausible_heap_addr);
            /* If this is a big object, count it as                     */
            /* size/256 + 1 objects.                                    */
            ++i;
            if ((descr & GC_DS_TAGS) == GC_DS_LENGTH) i += (int)(descr >> 8);
        }
eg's avatar
eg committed
987 988 989 990 991
    }
    *next = p;
    return top;
}

992 993
/* Copy back a local mark stack.        */
/* low and high are inclusive bounds.   */
994
STATIC void GC_return_mark_stack(mse * low, mse * high)
eg's avatar
eg committed
995 996 997 998 999
{
    mse * my_top;
    mse * my_start;
    size_t stack_size;

1000
    if ((word)high < (word)low) return;
eg's avatar
eg committed
1001 1002
    stack_size = high - low + 1;
    GC_acquire_mark_lock();
1003
    my_top = GC_mark_stack_top; /* Concurrent modification impossible. */
eg's avatar
eg committed
1004
    my_start = my_top + 1;
1005 1006 1007
    if ((word)(my_start - GC_mark_stack + stack_size)
                > (word)GC_mark_stack_size) {
      GC_COND_LOG_PRINTF("No room to copy back mark stack\n");
eg's avatar
eg committed
1008 1009
      GC_mark_state = MS_INVALID;
      GC_mark_stack_too_small = TRUE;
1010
      /* We drop the local mark stack.  We'll fix things later. */
eg's avatar
eg committed
1011 1012
    } else {
      BCOPY(low, my_start, stack_size * sizeof(mse));
eg's avatar
eg committed
1013
      GC_ASSERT((mse *)AO_load((volatile AO_t *)(&GC_mark_stack_top))
1014
                == my_top);
eg's avatar
eg committed
1015
      AO_store_release_write((volatile AO_t *)(&GC_mark_stack_top),
1016 1017
                             (AO_t)(my_top + stack_size));
                /* Ensures visibility of previously written stack contents. */
eg's avatar
eg committed
1018 1019 1020 1021 1022
    }
    GC_release_mark_lock();
    GC_notify_all_marker();
}

1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039
/* This function is only called when the local  */
/* and the main mark stacks are both empty.     */
static GC_bool has_inactive_helpers(void)
{
  GC_bool res;

  GC_acquire_mark_lock();
  res = GC_active_count < GC_helper_count;
  GC_release_mark_lock();
  return res;
}

/* Mark from the local mark stack.              */
/* On return, the local mark stack is empty.    */
/* But this may be achieved by copying the      */
/* local mark stack back into the global one.   */
/* We do not hold the mark lock.                */
1040
STATIC void GC_do_local_mark(mse *local_mark_stack, mse *local_top)
eg's avatar
eg committed
1041 1042 1043 1044 1045 1046
{
    unsigned n;
#   define N_LOCAL_ITERS 1

    for (;;) {
        for (n = 0; n < N_LOCAL_ITERS; ++n) {
1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073
            local_top = GC_mark_from(local_top, local_mark_stack,
                                     local_mark_stack + LOCAL_MARK_STACK_SIZE);
            if ((word)local_top < (word)local_mark_stack) return;
            if ((word)(local_top - local_mark_stack)
                        >= LOCAL_MARK_STACK_SIZE / 2) {
                GC_return_mark_stack(local_mark_stack, local_top);
                return;
            }
        }
        if ((word)AO_load((volatile AO_t *)&GC_mark_stack_top)
            < (word)AO_load(&GC_first_nonempty)
            && (word)local_top > (word)(local_mark_stack + 1)
            && has_inactive_helpers()) {
            /* Try to share the load, since the main stack is empty,    */
            /* and helper threads are waiting for a refill.             */
            /* The entries near the bottom of the stack are likely      */
            /* to require more work.  Thus we return those, even though */
            /* it's harder.                                             */
            mse * new_bottom = local_mark_stack
                                + (local_top - local_mark_stack)/2;
            GC_ASSERT((word)new_bottom > (word)local_mark_stack
                      && (word)new_bottom < (word)local_top);
            GC_return_mark_stack(local_mark_stack, new_bottom - 1);
            memmove(local_mark_stack, new_bottom,
                    (local_top - new_bottom + 1) * sizeof(mse));
            local_top -= (new_bottom - local_mark_stack);
        }
eg's avatar
eg committed
1074 1075 1076 1077 1078
    }
}

#define ENTRIES_TO_GET 5

1079 1080 1081 1082 1083
/* Mark using the local mark stack until the global mark stack is empty */
/* and there are no active workers. Update GC_first_nonempty to reflect */
/* progress.  Caller holds the mark lock.                               */
/* Caller has already incremented GC_helper_count.  We decrement it,    */
/* and maintain GC_active_count.                                        */
1084
STATIC void GC_mark_local(mse *local_mark_stack, int id)
eg's avatar
eg committed
1085 1086 1087 1088
{
    mse * my_first_nonempty;

    GC_active_count++;
eg's avatar
eg committed
1089
    my_first_nonempty = (mse *)AO_load(&GC_first_nonempty);
1090 1091 1092 1093
    GC_ASSERT((word)GC_mark_stack <= (word)my_first_nonempty);
    GC_ASSERT((word)my_first_nonempty
        <= (word)AO_load((volatile AO_t *)&GC_mark_stack_top) + sizeof(mse));
    GC_VERBOSE_LOG_PRINTF("Starting mark helper %d\n", id);
eg's avatar
eg committed
1094 1095
    GC_release_mark_lock();
    for (;;) {
1096
        size_t n_on_stack;
1097
        unsigned n_to_get;
1098 1099
        mse * my_top;
        mse * local_top;
eg's avatar
eg committed
1100
        mse * global_first_nonempty = (mse *)AO_load(&GC_first_nonempty);
eg's avatar
eg committed
1101

1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120
        GC_ASSERT((word)my_first_nonempty >= (word)GC_mark_stack &&
                  (word)my_first_nonempty <=
                        (word)AO_load((volatile AO_t *)&GC_mark_stack_top)
                        + sizeof(mse));
        GC_ASSERT((word)global_first_nonempty >= (word)GC_mark_stack);
        if ((word)my_first_nonempty < (word)global_first_nonempty) {
            my_first_nonempty = global_first_nonempty;
        } else if ((word)global_first_nonempty < (word)my_first_nonempty) {
            (void)AO_compare_and_swap(&GC_first_nonempty,
                                      (AO_t)global_first_nonempty,
                                      (AO_t)my_first_nonempty);
            /* If this fails, we just go ahead, without updating        */
            /* GC_first_nonempty.                                       */
        }
        /* Perhaps we should also update GC_first_nonempty, if it */
        /* is less.  But that would require using atomic updates. */
        my_top = (mse *)AO_load_acquire((volatile AO_t *)(&GC_mark_stack_top));
        if ((word)my_top < (word)my_first_nonempty) {
            GC_acquire_mark_lock();
eg's avatar
eg committed
1121
            my_top = GC_mark_stack_top;
1122 1123
                /* Asynchronous modification impossible here,   */
                /* since we hold mark lock.                     */
eg's avatar
eg committed
1124
            n_on_stack = my_top - my_first_nonempty + 1;
1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176
            if (0 == n_on_stack) {
                GC_active_count--;
                GC_ASSERT(GC_active_count <= GC_helper_count);
                /* Other markers may redeposit objects  */
                /* on the stack.                                */
                if (0 == GC_active_count) GC_notify_all_marker();
                while (GC_active_count > 0
                       && (word)AO_load(&GC_first_nonempty)
                                > (word)GC_mark_stack_top) {
                    /* We will be notified if either GC_active_count    */
                    /* reaches zero, or if more objects are pushed on   */
                    /* the global mark stack.                           */
                    GC_wait_marker();
                }
                if (GC_active_count == 0
                    && (word)AO_load(&GC_first_nonempty)
                        > (word)GC_mark_stack_top) {
                    GC_bool need_to_notify = FALSE;
                    /* The above conditions can't be falsified while we */
                    /* hold the mark lock, since neither                */
                    /* GC_active_count nor GC_mark_stack_top can        */
                    /* change.  GC_first_nonempty can only be           */
                    /* incremented asynchronously.  Thus we know that   */
                    /* both conditions actually held simultaneously.    */
                    GC_helper_count--;
                    if (0 == GC_helper_count) need_to_notify = TRUE;
                    GC_VERBOSE_LOG_PRINTF("Finished mark helper %d\n", id);
                    if (need_to_notify) GC_notify_all_marker();
                    return;
                }
                /* else there's something on the stack again, or        */
                /* another helper may push something.                   */
                GC_active_count++;
                GC_ASSERT(GC_active_count > 0);
                GC_release_mark_lock();
                continue;
            } else {
                GC_release_mark_lock();
            }
        } else {
            n_on_stack = my_top - my_first_nonempty + 1;
        }
        n_to_get = ENTRIES_TO_GET;
        if (n_on_stack < 2 * ENTRIES_TO_GET) n_to_get = 1;
        local_top = GC_steal_mark_stack(my_first_nonempty, my_top,
                                        local_mark_stack, n_to_get,
                                        &my_first_nonempty);
        GC_ASSERT((word)my_first_nonempty >= (word)GC_mark_stack &&
                  (word)my_first_nonempty <=
                        (word)AO_load((volatile AO_t *)&GC_mark_stack_top)
                        + sizeof(mse));
        GC_do_local_mark(local_mark_stack, local_top);
eg's avatar
eg committed
1177 1178 1179
    }
}

1180 1181 1182 1183
/* Perform Parallel mark.                       */
/* We hold the GC lock, not the mark lock.      */
/* Currently runs until the mark stack is       */
/* empty.                                       */
1184
STATIC void GC_do_parallel_mark(void)
eg's avatar
eg committed
1185 1186
{
    mse local_mark_stack[LOCAL_MARK_STACK_SIZE];
1187
                /* Note: local_mark_stack is quite big (up to 128 KiB). */
eg's avatar
eg committed
1188 1189 1190

    GC_acquire_mark_lock();
    GC_ASSERT(I_HOLD_LOCK());
1191 1192
    /* This could be a GC_ASSERT, but it seems safer to keep it on      */
    /* all the time, especially since it's cheap.                       */
eg's avatar
eg committed
1193
    if (GC_help_wanted || GC_active_count != 0 || GC_helper_count != 0)
1194 1195 1196
        ABORT("Tried to start parallel mark in bad state");
    GC_VERBOSE_LOG_PRINTF("Starting marking for mark phase number %lu\n",
                          (unsigned long)GC_mark_no);
eg's avatar
eg committed
1197
    GC_first_nonempty = (AO_t)GC_mark_stack;
eg's avatar
eg committed
1198 1199 1200 1201
    GC_active_count = 0;
    GC_helper_count = 1;
    GC_help_wanted = TRUE;
    GC_notify_all_marker();
1202
        /* Wake up potential helpers.   */
eg's avatar
eg committed
1203 1204
    GC_mark_local(local_mark_stack, 0);
    GC_help_wanted = FALSE;
1205 1206 1207 1208
    /* Done; clean up.  */
    while (GC_helper_count > 0) {
      GC_wait_marker();
    }
eg's avatar
eg committed
1209
    /* GC_helper_count cannot be incremented while GC_help_wanted == FALSE */
1210 1211
    GC_VERBOSE_LOG_PRINTF("Finished marking for mark phase number %lu\n",
                          (unsigned long)GC_mark_no);
eg's avatar
eg committed
1212 1213 1214 1215 1216 1217
    GC_mark_no++;
    GC_release_mark_lock();
    GC_notify_all_marker();
}


1218 1219 1220 1221
/* Try to help out the marker, if it's running.         */
/* We do not hold the GC lock, but the requestor does.  */
/* And we hold the mark lock.                           */
GC_INNER void GC_help_marker(word my_mark_no)
eg's avatar
eg committed
1222
{
1223 1224
#   define my_id my_id_mse.mse_descr.w
    mse my_id_mse;  /* align local_mark_stack explicitly */
eg's avatar
eg committed
1225
    mse local_mark_stack[LOCAL_MARK_STACK_SIZE];
1226
                /* Note: local_mark_stack is quite big (up to 128 KiB). */
eg's avatar
eg committed
1227

1228
    GC_ASSERT(GC_parallel);
eg's avatar
eg committed
1229
    while (GC_mark_no < my_mark_no
eg's avatar
eg committed
1230
           || (!GC_help_wanted && GC_mark_no == my_mark_no)) {
eg's avatar
eg committed
1231 1232 1233
      GC_wait_marker();
    }
    my_id = GC_helper_count;
1234 1235 1236
    if (GC_mark_no != my_mark_no || my_id > (unsigned)GC_markers_m1) {
      /* Second test is useful only if original threads can also        */
      /* act as helpers.  Under Linux they can't.                       */
eg's avatar
eg committed
1237 1238
      return;
    }
1239 1240
    GC_helper_count = (unsigned)my_id + 1;
    GC_mark_local(local_mark_stack, (int)my_id);
eg's avatar
eg committed
1241
    /* GC_mark_local decrements GC_helper_count. */
1242
#   undef my_id
eg's avatar
eg committed
1243 1244 1245 1246
}

#endif /* PARALLEL_MARK */

1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267
GC_INNER void GC_scratch_recycle_inner(void *ptr, size_t bytes)
{
  if (ptr != NULL) {
    size_t page_offset = (word)ptr & (GC_page_size - 1);
    size_t displ = 0;
    size_t recycled_bytes;

    GC_ASSERT(bytes != 0);
    GC_ASSERT(GC_page_size != 0);
    /* TODO: Assert correct memory flags if GWW_VDB */
    if (page_offset != 0)
      displ = GC_page_size - page_offset;
    recycled_bytes = (bytes - displ) & ~(GC_page_size - 1);
    GC_COND_LOG_PRINTF("Recycle %lu/%lu scratch-allocated bytes at %p\n",
                       (unsigned long)recycled_bytes, (unsigned long)bytes,
                       ptr);
    if (recycled_bytes > 0)
      GC_add_to_heap((struct hblk *)((word)ptr + displ), recycled_bytes);
  }
}

eg's avatar
eg committed
1268
/* Allocate or reallocate space for mark stack of size n entries.  */
1269
/* May silently fail.                                              */
eg's avatar
eg committed
1270
static void alloc_mark_stack(size_t n)
eg's avatar
eg committed
1271 1272
{
    mse * new_stack = (mse *)GC_scratch_alloc(n * sizeof(struct GC_ms_entry));
1273
#   ifdef GWW_VDB
1274 1275 1276
      /* Don't recycle a stack segment obtained with the wrong flags.   */
      /* Win32 GetWriteWatch requires the right kind of memory.         */
      static GC_bool GC_incremental_at_stack_alloc = FALSE;
1277 1278 1279 1280
      GC_bool recycle_old = (!GC_incremental || GC_incremental_at_stack_alloc);

      GC_incremental_at_stack_alloc = GC_incremental;
#   else
1281
#     define recycle_old TRUE
1282
#   endif
1283

eg's avatar
eg committed
1284
    GC_mark_stack_too_small = FALSE;
1285
    if (GC_mark_stack != NULL) {
eg's avatar
eg committed
1286
        if (new_stack != 0) {
1287
          if (recycle_old) {
1288
            /* Recycle old space */
1289 1290 1291
            GC_scratch_recycle_inner(GC_mark_stack,
                        GC_mark_stack_size * sizeof(struct GC_ms_entry));
          }
eg's avatar
eg committed
1292 1293
          GC_mark_stack = new_stack;
          GC_mark_stack_size = n;
1294 1295 1296 1297
          /* FIXME: Do we need some way to reset GC_mark_stack_size?    */
          GC_mark_stack_limit = new_stack + n;
          GC_COND_LOG_PRINTF("Grew mark stack to %lu frames\n",
                             (unsigned long)GC_mark_stack_size);
eg's avatar
eg committed
1298
        } else {
1299
          WARN("Failed to grow mark stack to %" WARN_PRIdPTR " frames\n", n);
eg's avatar
eg committed
1300 1301 1302
        }
    } else {
        if (new_stack == 0) {
eg's avatar
eg committed
1303
            GC_err_printf("No space for mark stack\n");
eg's avatar
eg committed
1304 1305 1306 1307
            EXIT();
        }
        GC_mark_stack = new_stack;
        GC_mark_stack_size = n;
1308
        GC_mark_stack_limit = new_stack + n;
eg's avatar
eg committed
1309 1310 1311 1312
    }
    GC_mark_stack_top = GC_mark_stack-1;
}

1313
GC_INNER void GC_mark_init(void)
eg's avatar
eg committed
1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324
{
    alloc_mark_stack(INITIAL_MARK_STACK_SIZE);
}

/*
 * Push all locations between b and t onto the mark stack.
 * b is the first location to be checked. t is one past the last
 * location to be checked.
 * Should only be used if there is no possibility of mark stack
 * overflow.
 */
1325
GC_API void GC_CALL GC_push_all(char *bottom, char *top)
eg's avatar
eg committed
1326 1327
{
    register word length;
1328 1329 1330 1331 1332

    bottom = (char *)(((word) bottom + ALIGNMENT-1) & ~(ALIGNMENT-1));
    top = (char *)(((word) top) & ~(ALIGNMENT-1));
    if ((word)bottom >= (word)top) return;

eg's avatar
eg committed
1333
    GC_mark_stack_top++;
1334 1335
    if ((word)GC_mark_stack_top >= (word)GC_mark_stack_limit) {
        ABORT("Unexpected mark stack overflow");
eg's avatar
eg committed
1336 1337 1338
    }
    length = top - bottom;
#   if GC_DS_TAGS > ALIGNMENT - 1
1339 1340
        length += GC_DS_TAGS;
        length &= ~GC_DS_TAGS;
eg's avatar
eg committed
1341
#   endif
eg's avatar
eg committed
1342
    GC_mark_stack_top -> mse_start = bottom;
1343
    GC_mark_stack_top -> mse_descr.w = length;
eg's avatar
eg committed
1344 1345
}

1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358
#ifndef GC_DISABLE_INCREMENTAL

  /* Analogous to the above, but push only those pages h with           */
  /* dirty_fn(h) != 0.  We use GC_push_all to actually push the block.  */
  /* Used both to selectively push dirty pages, or to push a block in   */
  /* piecemeal fashion, to allow for more marking concurrency.          */
  /* Will not overflow mark stack if GC_push_all pushes a small fixed   */
  /* number of entries.  (This is invoked only if GC_push_all pushes    */
  /* a single entry, or if it marks each object before pushing it, thus */
  /* ensuring progress in the event of a stack overflow.)               */
  STATIC void GC_push_selected(ptr_t bottom, ptr_t top,
                               GC_bool (*dirty_fn)(struct hblk *))
  {
eg's avatar
eg committed
1359
    struct hblk * h;
eg's avatar
eg committed
1360

1361 1362
    bottom = (ptr_t)(((word) bottom + ALIGNMENT-1) & ~(ALIGNMENT-1));
    top = (ptr_t)(((word) top) & ~(ALIGNMENT-1));
1363
    if ((word)bottom >= (word)top) return;
eg's avatar
eg committed
1364 1365

    h = HBLKPTR(bottom + HBLKSIZE);
1366 1367 1368 1369 1370
    if ((word)top <= (word)h) {
        if ((*dirty_fn)(h-1)) {
            GC_push_all(bottom, top);
        }
        return;
eg's avatar
eg committed
1371 1372
    }
    if ((*dirty_fn)(h-1)) {
1373
        GC_push_all(bottom, (ptr_t)h);
eg's avatar
eg committed
1374
    }
1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385

    while ((word)(h+1) <= (word)top) {
        if ((*dirty_fn)(h)) {
            if ((word)(GC_mark_stack_top - GC_mark_stack)
                > 3 * GC_mark_stack_size / 4) {
                /* Danger of mark stack overflow */
                GC_push_all((ptr_t)h, top);
                return;
            } else {
                GC_push_all((ptr_t)h, (ptr_t)(h+1));
            }
eg's avatar
eg committed
1386
        }
1387
        h++;
eg's avatar
eg committed
1388 1389
    }

1390 1391 1392 1393 1394 1395 1396
    if ((ptr_t)h != top && (*dirty_fn)(h)) {
       GC_push_all((ptr_t)h, top);
    }
    if ((word)GC_mark_stack_top >= (word)GC_mark_stack_limit) {
        ABORT("Unexpected mark stack overflow");
    }
  }
eg's avatar
eg committed
1397

1398 1399 1400 1401
  GC_API void GC_CALL GC_push_conditional(char *bottom, char *top, int all)
  {
    if (!all) {
      GC_push_selected((ptr_t)bottom, (ptr_t)top, GC_page_was_dirty);
eg's avatar
eg committed
1402
    } else {
1403 1404 1405 1406 1407 1408 1409 1410 1411
#     ifdef PROC_VDB
        if (GC_incremental) {
          /* Pages that were never dirtied cannot contain pointers.     */
          GC_push_selected((ptr_t)bottom, (ptr_t)top, GC_page_was_ever_dirty);
        } else
#     endif
      /* else */ {
        GC_push_all(bottom, top);
      }
eg's avatar
eg committed
1412
    }
1413 1414 1415 1416 1417 1418 1419 1420
  }
#else
  GC_API void GC_CALL GC_push_conditional(char *bottom, char *top,
                                          int all GC_ATTR_UNUSED)
  {
    GC_push_all(bottom, top);
  }
#endif /* GC_DISABLE_INCREMENTAL */
eg's avatar
eg committed
1421

1422
#if defined(MSWIN32) || defined(MSWINCE)
eg's avatar
eg committed
1423
  void __cdecl GC_push_one(word p)
1424
#else
eg's avatar
eg committed
1425
  void GC_push_one(word p)
1426
#endif
eg's avatar
eg committed
1427
{
1428
    GC_PUSH_ONE_STACK(p, MARKED_FROM_REGISTER);
eg's avatar
eg committed
1429 1430
}

1431 1432 1433 1434
GC_API struct GC_ms_entry * GC_CALL GC_mark_and_push(void *obj,
                                                mse *mark_stack_ptr,
                                                mse *mark_stack_limit,
                                                void ** src GC_ATTR_UNUSED)
eg's avatar
eg committed
1435
{
eg's avatar
eg committed
1436 1437 1438 1439
    hdr * hhdr;

    PREFETCH(obj);
    GET_HDR(obj, hhdr);
1440 1441 1442 1443 1444 1445
    if ((EXPECT(IS_FORWARDING_ADDR_OR_NIL(hhdr), FALSE)
         && (!GC_all_interior_pointers
             || NULL == (hhdr = GC_find_header(GC_base(obj)))))
        || EXPECT(HBLK_IS_FREE(hhdr), FALSE)) {
      GC_ADD_TO_BLACK_LIST_NORMAL(obj, (ptr_t)src);
      return mark_stack_ptr;
eg's avatar
eg committed
1446
    }
eg's avatar
eg committed
1447

eg's avatar
eg committed
1448
    PUSH_CONTENTS_HDR(obj, mark_stack_ptr /* modified */, mark_stack_limit,
1449
                      (ptr_t)src, hhdr, TRUE);
eg's avatar
eg committed
1450 1451
    return mark_stack_ptr;
}
eg's avatar
eg committed
1452

1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463
#if defined(MANUAL_VDB) && defined(THREADS)
  void GC_dirty(ptr_t p);
#endif

/* Mark and push (i.e. gray) a single object p onto the main    */
/* mark stack.  Consider p to be valid if it is an interior     */
/* pointer.                                                     */
/* The object p has passed a preliminary pointer validity       */
/* test, but we do not definitely know whether it is valid.     */
/* Mark bits are NOT atomically updated.  Thus this must be the */
/* only thread setting them.                                    */
eg's avatar
eg committed
1464
# if defined(PRINT_BLACK_LIST) || defined(KEEP_BACK_PTRS)
1465
    GC_INNER void GC_mark_and_push_stack(ptr_t p, ptr_t source)
eg's avatar
eg committed
1466
# else
1467
    GC_INNER void GC_mark_and_push_stack(ptr_t p)
1468
#   define source ((ptr_t)0)
eg's avatar
eg committed
1469 1470
# endif
{
1471
    hdr * hhdr;
eg's avatar
eg committed
1472
    ptr_t r = p;
1473

eg's avatar
eg committed
1474
    PREFETCH(p);
eg's avatar
eg committed
1475
    GET_HDR(p, hhdr);
1476 1477 1478 1479 1480 1481
    if (EXPECT(IS_FORWARDING_ADDR_OR_NIL(hhdr), FALSE)
        && (NULL == hhdr
            || (r = GC_base(p)) == NULL
            || (hhdr = HDR(r)) == NULL)) {
        GC_ADD_TO_BLACK_LIST_STACK(p, source);
        return;
eg's avatar
eg committed
1482
    }
1483 1484 1485
    if (EXPECT(HBLK_IS_FREE(hhdr), FALSE)) {
        GC_ADD_TO_BLACK_LIST_NORMAL(p, source);
        return;
eg's avatar
eg committed
1486 1487
    }
#   if defined(MANUAL_VDB) && defined(THREADS)
1488 1489 1490
      /* Pointer is on the stack.  We may have dirtied the object       */
      /* it points to, but not yet have called GC_dirty();              */
      GC_dirty(p);      /* Implicitly affects entire object.            */
eg's avatar
eg committed
1491 1492
#   endif
    PUSH_CONTENTS_HDR(r, GC_mark_stack_top, GC_mark_stack_limit,
1493 1494 1495 1496 1497
                      source, hhdr, FALSE);
    /* We silently ignore pointers to near the end of a block,  */
    /* which is very mildly suboptimal.                         */
    /* FIXME: We should probably add a header word to address   */
    /* this.                                                    */
eg's avatar
eg committed
1498
}
1499
# undef source
eg's avatar
eg committed
1500

1501
#ifdef TRACE_BUF
eg's avatar
eg committed
1502

1503 1504 1505
# ifndef TRACE_ENTRIES
#   define TRACE_ENTRIES 1000
# endif
eg's avatar
eg committed
1506 1507 1508 1509

struct trace_entry {
    char * kind;
    word gc_no;
eg's avatar
eg committed
1510
    word bytes_allocd;
eg's avatar
eg committed
1511 1512 1513 1514 1515 1516 1517 1518 1519 1520
    word arg1;
    word arg2;
} GC_trace_buf[TRACE_ENTRIES];

int GC_trace_buf_ptr = 0;

void GC_add_trace_entry(char *kind, word arg1, word arg2)
{
    GC_trace_buf[GC_trace_buf_ptr].kind = kind;
    GC_trace_buf[GC_trace_buf_ptr].gc_no = GC_gc_no;
eg's avatar
eg committed
1521
    GC_trace_buf[GC_trace_buf_ptr].bytes_allocd = GC_bytes_allocd;
eg's avatar
eg committed
1522 1523 1524 1525 1526 1527
    GC_trace_buf[GC_trace_buf_ptr].arg1 = arg1 ^ 0x80000000;
    GC_trace_buf[GC_trace_buf_ptr].arg2 = arg2 ^ 0x80000000;
    GC_trace_buf_ptr++;
    if (GC_trace_buf_ptr >= TRACE_ENTRIES) GC_trace_buf_ptr = 0;
}

1528
GC_API void GC_CALL GC_print_trace_inner(word gc_no)
eg's avatar
eg committed
1529 1530
{
    int i;
1531

eg's avatar
eg committed
1532
    for (i = GC_trace_buf_ptr-1; i != GC_trace_buf_ptr; i--) {
1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543
        struct trace_entry *p;

        if (i < 0) i = TRACE_ENTRIES-1;
        p = GC_trace_buf + i;
        if (p -> gc_no < gc_no || p -> kind == 0) {
            return;
        }
        GC_printf("Trace:%s (gc:%u, bytes:%lu) 0x%lX, 0x%lX\n",
                  p -> kind, (unsigned)p -> gc_no,
                  (unsigned long)p -> bytes_allocd,
                  (long)p->arg1 ^ 0x80000000L, (long)p->arg2 ^ 0x80000000L);
eg's avatar
eg committed
1544
    }
1545 1546 1547 1548 1549 1550 1551 1552 1553 1554
    GC_printf("Trace incomplete\n");
}

GC_API void GC_CALL GC_print_trace(word gc_no)
{
    DCL_LOCK_STATE;

    LOCK();
    GC_print_trace_inner(gc_no);
    UNLOCK();
eg's avatar
eg committed
1555 1556
}

1557
#endif /* TRACE_BUF */
eg's avatar
eg committed
1558 1559 1560 1561 1562 1563

/*
 * A version of GC_push_all that treats all interior pointers as valid
 * and scans the entire region immediately, in case the contents
 * change.
 */
1564 1565
GC_ATTR_NO_SANITIZE_ADDR GC_ATTR_NO_SANITIZE_MEMORY
GC_API void GC_CALL GC_push_all_eager(char *bottom, char *top)
eg's avatar
eg committed
1566
{