DualPointer ptr;

size_t capacity;

size_t available;

CUevent event;

Chunk* next;

bool open_transaction;

Chunk* head;

Chunk* transaction_head;

Chunk* tail;

StreamBuffer* next_free;

Vec<unsigned long long> stream_ids;

Vec<StreamBuffer*> stream_buffers;

size_t cur_chunk_capacity = kInitialChunkCapacity;

Chunk* chunk_freelist = nullptr;

StreamBuffer* sb_freelist = nullptr;

dual_ptr_free(driver, chunk->ptr);

driver->cuEventDestroy(chunk->event);

delete chunk;

if (chunk->capacity == pool->cur_chunk_capacity) {

chunk->next = pool->chunk_freelist;

pool->chunk_freelist = chunk;

} else {

delete_chunk(driver, chunk);

}

while (sb->head != sb->transaction_head) {

Chunk* chunk = sb->head;

if (driver->cuEventQuery(chunk->event) != CUDA_SUCCESS)

break;

sb->head = chunk->next;

if (!sb->head) sb->tail = nullptr;

reclaim_chunk(driver, chunk, pool);

}

size_t i = 0;

size_t n = pool->stream_buffers.size();

StreamBuffer** stream_buffers = pool->stream_buffers.data();

while (i < n) {

StreamBuffer* sb = stream_buffers[i];

poll_events(driver, sb, pool);

if (!sb->head && !sb->open_transaction) {

sb->next_free = pool->sb_freelist;

pool->sb_freelist = sb;

--n;

stream_buffers[i] = stream_buffers[n];

pool->stream_ids[i] = pool->stream_ids[n];

} else {

++i;

}

}

pool->stream_ids.resize(n);

pool->stream_buffers.resize(n);

for (Chunk *chunk = pool->chunk_freelist, *next; chunk; chunk = next) {

next = chunk->next;

delete_chunk(driver, chunk);

}

pool->chunk_freelist = nullptr;

for (size_t n = stream_ids.size(), i = 0; i < n; ++i) {

if (stream_ids[i] == stream_id)

return i;

}

return SIZE_MAX;

StreamBufferPool* pool,

CUstream stream) {

unsigned long long stream_id;

if (driver->cuStreamGetId(stream, &stream_id) != CUDA_SUCCESS)

return {};

const Vec<unsigned long long>& stream_ids = pool->stream_ids;

StreamBuffer* sb;

size_t i = find_stream(stream_ids, stream_id);

if (i == SIZE_MAX) {

sb = alloc_stream_buffer(pool);

pool->stream_ids.push_back(stream_id);

pool->stream_buffers.push_back(sb);

} else {

sb = pool->stream_buffers[i];

}

CHECK(!sb->open_transaction);

sb->open_transaction = true;

sb->transaction_head = nullptr;

return StreamBufferTransaction(pool, sb, stream, driver);

StreamBufferPool* pool = this->pool_;

StreamBuffer* sb = this->sb_;

const DriverApi* driver = this->driver_;

if (size >= SIZE_MAX - kAlignment)

return {};

size = (size + kAlignment - 1) & ~static_cast<size_t>(kAlignment - 1);

Chunk* chunk = sb->tail;

if (!chunk || chunk->available < size) {

chunk = allocate_chunk(driver, pool, sb, size);

if (!chunk) return {};

if (sb->tail)

sb->tail->next = chunk;

else

sb->head = chunk;

sb->tail = chunk;

}

if (!sb->transaction_head)

sb->transaction_head = chunk;

size_t pos = chunk->available -= size;

return dualptr_offset(chunk->ptr, pos);

StreamBuffer* sb = this->sb_;

if (!sb) return;

CUstream stream = this->stream_;

const DriverApi* driver = this->driver_;

this->pool_ = nullptr;

this->sb_ = nullptr;

this->stream_ = {};

for (Chunk* chunk = sb->transaction_head; chunk; chunk = chunk->next) {

CUresult res = driver->cuEventRecord(chunk->event, stream);

CHECK(res == CUDA_SUCCESS); // TODO: is there a more graceful way to recover?

}

sb->open_transaction = false;

sb->transaction_head = nullptr;