| // SPDX-License-Identifier: GPL-2.0 OR Apache-2.0 | |
| /* | |
| * Copyright 2020 Google LLC | |
| * | |
| * Licensed under the Apache License, Version 2.0 (the "License"); | |
| * you may not use this file except in compliance with the License. | |
| * You may obtain a copy of the License at | |
| * | |
| * https://www.apache.org/licenses/LICENSE-2.0 | |
| * | |
| * Unless required by applicable law or agreed to in writing, software | |
| * distributed under the License is distributed on an "AS IS" BASIS, | |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| * See the License for the specific language governing permissions and | |
| * limitations under the License. | |
| */ | |
| #include "aoc_ipc_core.h" | |
| #include "aoc_ipc_core_internal.h" | |
| /* | |
| * Due to this file being shared between the linux kernel and EFW with | |
| * very different coding styles, disable automatic formatting | |
| */ | |
| #define AOC_ASSERT(x) | |
| #define VALID_DIRECTION(d) ((d == AOC_UP) || (d == AOC_DOWN)) | |
| #if __KERNEL__ | |
| #include <linux/io.h> | |
| #define copy_to_buffer(dst, src, len) memcpy_toio(dst, src, len) | |
| #define copy_from_buffer(dst, src, len) memcpy_fromio(dst, src, len) | |
| #else | |
| #define copy_to_buffer(dst, src, len) memcpy(dst, src, len) | |
| #define copy_from_buffer(dst, src, len) memcpy(dst, src, len) | |
| static inline void iowrite32(u32 value, void *addr) | |
| { | |
| volatile u32 *r = addr; | |
| *r = value; | |
| } | |
| static inline u32 ioread32(void *addr) | |
| { | |
| volatile u32 *r = addr; | |
| return *r; | |
| } | |
| #define EXPORT_SYMBOL(x) | |
| #endif | |
| static inline void _region_advance_tx(struct aoc_ipc_memory_region *r, | |
| u32 amount) | |
| { | |
| u32 sz = ioread32(&r->size); | |
| u32 tx = ioread32(&r->tx); | |
| u32 wp = ioread32(&r->wp); | |
| u32 updated_tx = (tx + amount); | |
| u32 updated_wp = (wp + amount) % sz; | |
| iowrite32(updated_tx, &r->tx); | |
| iowrite32(updated_wp, &r->wp); | |
| } | |
| static inline void _region_advance_rx(struct aoc_ipc_memory_region *r, | |
| u32 amount) | |
| { | |
| u32 sz = ioread32(&r->size); | |
| u32 rx = ioread32(&r->rx); | |
| u32 rp = ioread32(&r->rp); | |
| u32 updated_rx = (rx + amount); | |
| u32 updated_rp = (rp + amount) % sz; | |
| iowrite32(updated_rx, &r->rx); | |
| iowrite32(updated_rp, &r->rp); | |
| } | |
| static u32 _aoc_queue_write_buffer(u8 *dest, const u8 *src, u32 length); | |
| static u32 _aoc_ring_read_buffer(const u8 *ring, | |
| struct aoc_ipc_memory_region *r, | |
| u8 *dest, | |
| size_t *dest_size); | |
| static u32 _read_write_delta(struct aoc_ipc_memory_region *r) | |
| { | |
| u32 tx = ioread32(&r->tx); | |
| u32 rx = ioread32(&r->rx); | |
| if (tx >= rx) { | |
| return tx - rx; | |
| } else { | |
| // Here we are assuming that this is a wraparound and not a read | |
| // pointer that has been advanced past the write pointer or | |
| // a unsynced read pointer after a reset write pointer | |
| return (SIZE_MAX - rx) + tx + 1; | |
| } | |
| } | |
| static u32 _aoc_queue_write_buffer(u8 *dest, const u8 *src, u32 length) | |
| { | |
| struct aoc_ipc_message_header hdr = { | |
| .length = length, .r1 = 0, .r2 = 0, .r3 = 0 | |
| }; | |
| copy_to_buffer(dest, &hdr, sizeof(hdr)); | |
| dest += sizeof(hdr); | |
| copy_to_buffer(dest, src, length); | |
| return length; | |
| } | |
| static bool _aoc_ring_reader_sync_offset(struct aoc_ipc_memory_region *r, | |
| size_t bytes_to_leave) | |
| { | |
| u32 sz = r->size; | |
| u32 delta = _read_write_delta(r); | |
| if (bytes_to_leave > sz) | |
| bytes_to_leave = sz; | |
| if (delta < bytes_to_leave) | |
| bytes_to_leave = delta; | |
| _region_advance_rx(r, delta - bytes_to_leave); | |
| return true; | |
| } | |
| /* Returns the amount of bytes read from buffer */ | |
| static u32 _aoc_ring_read_buffer(const u8 *ring, | |
| struct aoc_ipc_memory_region *r, | |
| u8 *dest, | |
| size_t *dest_size) | |
| { | |
| u32 sz; | |
| u32 tx; | |
| u32 rx; | |
| u32 wp; | |
| u32 rp; | |
| u32 to_read; | |
| u32 bytes_available; | |
| bytes_available = _read_write_delta(r); | |
| sz = ioread32(&r->size); | |
| if (bytes_available > sz) | |
| _aoc_ring_reader_sync_offset(r, sz); | |
| tx = ioread32(&r->tx); | |
| rx = ioread32(&r->rx); | |
| wp = ioread32(&r->wp); | |
| rp = ioread32(&r->rp); | |
| if (wp > rp) { | |
| bytes_available = wp - rp; | |
| } else if (wp < rp) { | |
| bytes_available = wp + sz - rp; | |
| } else { | |
| if (tx == rx) | |
| bytes_available = 0; | |
| else | |
| bytes_available = sz; | |
| } | |
| if (bytes_available > *dest_size) | |
| to_read = *dest_size; | |
| else | |
| to_read = bytes_available; | |
| if (rp + to_read <= sz) { | |
| copy_from_buffer(dest, ring + rp, to_read); | |
| } else { | |
| u32 partial = sz - rp; | |
| copy_from_buffer(dest, ring + rp, partial); | |
| copy_from_buffer(dest + partial, ring, to_read - partial); | |
| } | |
| *dest_size = to_read; | |
| return to_read; | |
| } | |
| static u32 _aoc_ring_write_buffer(u8 *ring, | |
| struct aoc_ipc_memory_region *r, | |
| const u8 *src, | |
| size_t size) | |
| { | |
| u32 wp = ioread32(&r->wp); | |
| u32 sz = r->size; | |
| u32 to_write = size; | |
| if (to_write > sz) { | |
| /* Allow the write, but only commit the beginning */ | |
| to_write = sz; | |
| } | |
| if (wp + to_write <= sz) { | |
| copy_to_buffer(ring + wp, src, to_write); | |
| } else { | |
| size_t partial = sz - wp; | |
| copy_to_buffer(ring + wp, src, partial); | |
| copy_to_buffer(ring, src + partial, to_write - partial); | |
| } | |
| return to_write; | |
| } | |
| const char *aoc_service_name(aoc_service *service) | |
| { | |
| struct aoc_ipc_service_header *header = service; | |
| if (header->name[AOC_SERVICE_NAME_LENGTH - 1] != '\0') | |
| return NULL; | |
| return header->name; | |
| } | |
| static inline int aoc_service_type(aoc_service *service) | |
| { | |
| struct aoc_ipc_service_header *header = service; | |
| return (header->flags & AOC_SERVICE_TYPE_MASK) >> 16; | |
| } | |
| bool aoc_service_is_queue(aoc_service *service) | |
| { | |
| return aoc_service_type(service) == AOC_SERVICE_TYPE_QUEUE; | |
| } | |
| bool aoc_service_is_ring(aoc_service *service) | |
| { | |
| return aoc_service_type(service) == AOC_SERVICE_TYPE_RING; | |
| } | |
| bool aoc_service_is_buffer(aoc_service *service) | |
| { | |
| return aoc_service_type(service) == AOC_SERVICE_TYPE_BUFFER; | |
| } | |
| int aoc_service_irq_index(aoc_service *service) | |
| { | |
| struct aoc_ipc_service_header *header = service; | |
| return (header->flags & AOC_SERVICE_IRQ_MASK) >> AOC_SERVICE_IRQ_SHIFT; | |
| } | |
| size_t aoc_ring_bytes_read(aoc_service *service, aoc_direction dir) | |
| { | |
| struct aoc_ipc_service_header *header = service; | |
| struct aoc_ipc_memory_region *region; | |
| if (!service || !aoc_service_is_ring(service) || !VALID_DIRECTION(dir)) | |
| return 0; | |
| region = &header->regions[dir]; | |
| return ioread32(®ion->rx); | |
| } | |
| EXPORT_SYMBOL(aoc_ring_bytes_read); | |
| size_t aoc_ring_bytes_written(aoc_service *service, aoc_direction dir) | |
| { | |
| struct aoc_ipc_service_header *header = service; | |
| struct aoc_ipc_memory_region *region; | |
| if (!service || !aoc_service_is_ring(service) || !VALID_DIRECTION(dir)) | |
| return 0; | |
| region = &header->regions[dir]; | |
| return ioread32(®ion->tx); | |
| } | |
| EXPORT_SYMBOL(aoc_ring_bytes_written); | |
| bool aoc_ring_did_overflow(aoc_service *service, aoc_direction dir) | |
| { | |
| struct aoc_ipc_service_header *header = service; | |
| struct aoc_ipc_memory_region *region; | |
| if (!service || !aoc_service_is_ring(service) || !VALID_DIRECTION(dir)) | |
| return false; | |
| region = &header->regions[dir]; | |
| return (_read_write_delta(region) > region->size); | |
| } | |
| size_t aoc_service_message_size(aoc_service *service, aoc_direction dir) | |
| { | |
| struct aoc_ipc_service_header *header = service; | |
| struct aoc_ipc_memory_region *region; | |
| if (!header || !VALID_DIRECTION(dir)) | |
| return 0; | |
| region = &header->regions[dir]; | |
| if (aoc_service_is_ring(service) || aoc_service_is_buffer(service)) { | |
| return region->size; | |
| } else { | |
| const size_t header_size = | |
| sizeof(struct aoc_ipc_message_header); | |
| if (region->size <= header_size) | |
| return 0; | |
| return region->size - header_size; | |
| } | |
| } | |
| size_t aoc_service_message_slots(aoc_service *service, aoc_direction dir) | |
| { | |
| struct aoc_ipc_service_header *header = service; | |
| struct aoc_ipc_memory_region *region; | |
| if (!header || !VALID_DIRECTION(dir)) | |
| return 0; | |
| region = &header->regions[dir]; | |
| return ioread32(®ion->slots); | |
| } | |
| size_t aoc_service_total_size(aoc_service *service, aoc_direction dir) | |
| { | |
| struct aoc_ipc_service_header *header = service; | |
| struct aoc_ipc_memory_region *region; | |
| if (!header || !VALID_DIRECTION(dir)) | |
| return 0; | |
| region = &header->regions[dir]; | |
| return ioread32(®ion->slots) * region->size; | |
| } | |
| size_t aoc_ring_bytes_available_to_read(aoc_service *service, aoc_direction dir) | |
| { | |
| struct aoc_ipc_service_header *s; | |
| struct aoc_ipc_memory_region *region; | |
| size_t difference; | |
| if (!service || !VALID_DIRECTION(dir) || | |
| !(aoc_service_is_ring(service))) | |
| return 0; | |
| s = service; | |
| region = &s->regions[dir]; | |
| difference = _read_write_delta(region); | |
| return difference > region->size ? region->size : difference; | |
| } | |
| EXPORT_SYMBOL(aoc_ring_bytes_available_to_read); | |
| size_t aoc_ring_bytes_available_to_write(aoc_service *service, | |
| aoc_direction dir) | |
| { | |
| struct aoc_ipc_service_header *s; | |
| struct aoc_ipc_memory_region *region; | |
| if (!service || !VALID_DIRECTION(dir) || | |
| !(aoc_service_is_ring(service))) | |
| return 0; | |
| s = service; | |
| region = &s->regions[dir]; | |
| return region->size - aoc_ring_bytes_available_to_read(service, dir); | |
| } | |
| EXPORT_SYMBOL(aoc_ring_bytes_available_to_write); | |
| size_t aoc_service_slots_available_to_read(aoc_service *service, | |
| aoc_direction dir) | |
| { | |
| struct aoc_ipc_service_header *s = service; | |
| struct aoc_ipc_memory_region *region; | |
| if (!s || !VALID_DIRECTION(dir)) | |
| return 0; | |
| region = &s->regions[dir]; | |
| if (ioread32(®ion->slots) == 0) | |
| return 0; | |
| if (aoc_service_is_buffer(service)) | |
| return 0; | |
| if (aoc_service_is_ring(service)) { | |
| /* | |
| * Rings have one slot and the tx/rx counters specify bytes. If the | |
| * byte counters match, there is nothing to read | |
| */ | |
| return ioread32(®ion->tx) != ioread32(®ion->rx) ? 1 : 0; | |
| } else { | |
| size_t diff = _read_write_delta(region); | |
| /* | |
| * TODO : The API contract says that the difference will never be larger | |
| * than the # of slots on the channel. Figure out what to do if this | |
| * assumption is ever false. | |
| */ | |
| return diff > region->slots ? region->slots : diff; | |
| } | |
| } | |
| size_t aoc_service_slots_available_to_write(aoc_service *service, | |
| aoc_direction dir) | |
| { | |
| struct aoc_ipc_service_header *s = service; | |
| struct aoc_ipc_memory_region *region; | |
| if (!s || !VALID_DIRECTION(dir)) | |
| return 0; | |
| region = &s->regions[dir]; | |
| if (ioread32(®ion->slots) == 0) | |
| return 0; | |
| if (aoc_service_is_ring(service)) { | |
| return 1; | |
| } else if (aoc_service_is_buffer(service)) { | |
| return 0; | |
| } else { | |
| size_t diff = _read_write_delta(region); | |
| AOC_ASSERT(diff <= ioread32(®ion->slots)); | |
| return ioread32(®ion->slots) - diff; | |
| } | |
| } | |
| void *aoc_service_current_read_pointer(struct aoc_ipc_service_header *service, | |
| void *base, aoc_direction dir) | |
| { | |
| struct aoc_ipc_memory_region *region; | |
| u8 *ptr; | |
| u32 offset; | |
| if (!service || !base || !VALID_DIRECTION(dir)) | |
| return NULL; | |
| region = &service->regions[dir]; | |
| ptr = base; | |
| if (aoc_service_is_ring(service)) { | |
| offset = ioread32(®ion->rp); | |
| } else { | |
| offset = (region->size * | |
| (ioread32(®ion->rx) % region->slots)); | |
| } | |
| return ptr + region->offset + offset; | |
| } | |
| void *aoc_service_current_write_pointer(struct aoc_ipc_service_header *service, | |
| void *base, aoc_direction dir) | |
| { | |
| struct aoc_ipc_memory_region *region; | |
| u8 *ptr = base; | |
| u32 offset; | |
| if (!service || !base || !VALID_DIRECTION(dir)) | |
| return NULL; | |
| region = &service->regions[dir]; | |
| ptr = base; | |
| if (aoc_service_is_ring(service)) { | |
| offset = ioread32(®ion->wp); | |
| } else { | |
| offset = (region->size * | |
| (ioread32(®ion->tx) % region->slots)); | |
| } | |
| return ptr + region->offset + offset; | |
| } | |
| size_t aoc_service_current_message_size(aoc_service *service, void *base, | |
| aoc_direction dir) | |
| { | |
| struct aoc_ipc_message_header *hdr; | |
| if (!aoc_service_can_read_message(service, dir)) | |
| return 0; | |
| hdr = aoc_service_current_read_pointer(service, base, dir); | |
| return hdr->length; | |
| } | |
| void *aoc_service_current_message_pointer(aoc_service *service, void *base, | |
| aoc_direction dir) | |
| { | |
| uint8_t *ptr; | |
| if (!aoc_service_can_read_message(service, dir)) { | |
| return NULL; | |
| } | |
| ptr = aoc_service_current_read_pointer(service, base, dir); | |
| ptr += sizeof(struct aoc_ipc_message_header); | |
| return ptr; | |
| } | |
| bool aoc_service_increment_write_index(aoc_service *service, aoc_direction dir) | |
| { | |
| return aoc_service_advance_write_index(service, dir, 1); | |
| } | |
| bool aoc_service_increment_read_index(aoc_service *service, aoc_direction dir) | |
| { | |
| return aoc_service_advance_read_index(service, dir, 1); | |
| } | |
| bool aoc_service_advance_write_index(aoc_service *service, aoc_direction dir, | |
| u32 amount) | |
| { | |
| struct aoc_ipc_memory_region *region; | |
| struct aoc_ipc_service_header *s; | |
| size_t available; | |
| if (!service) | |
| return false; | |
| available = aoc_service_slots_available_to_write(service, dir); | |
| if (aoc_service_is_ring(service)) | |
| available = aoc_service_message_size(service, dir); | |
| if (available < amount) | |
| return false; | |
| s = service; | |
| region = &s->regions[dir]; | |
| _region_advance_tx(region, amount); | |
| return true; | |
| } | |
| EXPORT_SYMBOL_GPL(aoc_service_advance_write_index); | |
| bool aoc_service_advance_read_index(aoc_service *service, aoc_direction dir, | |
| u32 amount) | |
| { | |
| struct aoc_ipc_memory_region *region; | |
| struct aoc_ipc_service_header *s; | |
| u32 bytes_to_advance = amount; | |
| s = service; | |
| if (!s) | |
| return false; | |
| if (aoc_service_is_ring(service)) { | |
| u32 bytes_available = aoc_ring_bytes_available_to_read(service, dir); | |
| if (bytes_available < amount) | |
| bytes_to_advance = bytes_available; | |
| } else { | |
| if (aoc_service_slots_available_to_read(service, dir) < amount) | |
| return false; | |
| } | |
| region = &s->regions[dir]; | |
| _region_advance_rx(region, bytes_to_advance); | |
| return (bytes_to_advance == amount); | |
| } | |
| EXPORT_SYMBOL_GPL(aoc_service_advance_read_index); | |
| bool aoc_ring_flush_read_data(aoc_service *service, aoc_direction dir, | |
| size_t bytes_to_leave) | |
| { | |
| struct aoc_ipc_memory_region *r; | |
| struct aoc_ipc_service_header *s; | |
| s = service; | |
| if (!s || !aoc_service_is_ring(service)) | |
| return false; | |
| r = &s->regions[dir]; | |
| return _aoc_ring_reader_sync_offset(r, bytes_to_leave); | |
| } | |
| EXPORT_SYMBOL(aoc_ring_flush_read_data); | |
| /* | |
| * To reset the aoc ring write ptr to the beginning of the aoc ring buffer. | |
| */ | |
| bool aoc_ring_reset_write_pointer(aoc_service *service, aoc_direction dir) | |
| { | |
| struct aoc_ipc_memory_region *r; | |
| struct aoc_ipc_service_header *s; | |
| int bytes_remaining; | |
| u32 sz; | |
| u32 wp; | |
| if (!service || !aoc_service_is_ring(service)) | |
| return false; | |
| s = service; | |
| r = &s->regions[dir]; | |
| sz = ioread32(&r->size); | |
| wp = ioread32(&r->wp); | |
| bytes_remaining = sz - wp; | |
| if (bytes_remaining > 0) | |
| _region_advance_tx(r, bytes_remaining); | |
| return true; | |
| } | |
| EXPORT_SYMBOL_GPL(aoc_ring_reset_write_pointer); | |
| bool aoc_service_can_read_message(aoc_service *service, aoc_direction dir) | |
| { | |
| return aoc_service_slots_available_to_read(service, dir) > 0; | |
| } | |
| bool aoc_service_can_write_message(aoc_service *service, aoc_direction dir) | |
| { | |
| return aoc_service_slots_available_to_write(service, dir) > 0; | |
| } | |
| bool aoc_service_read_message(aoc_service *service, void *base, | |
| aoc_direction dir, void *buffer, size_t *size) | |
| { | |
| struct aoc_ipc_service_header *s = | |
| (struct aoc_ipc_service_header *)service; | |
| struct aoc_ipc_memory_region *r; | |
| struct aoc_ipc_message_header *hdr; | |
| u8 *ptr; | |
| if (!service || !base || !buffer || !size || !VALID_DIRECTION(dir)) | |
| return false; | |
| if (!aoc_service_can_read_message(service, dir)) | |
| return false; | |
| r = &s->regions[dir]; | |
| if (aoc_service_is_ring(service)) { | |
| u32 bytes_read; | |
| ptr = aoc_service_ring_base(service, base, dir); | |
| bytes_read = _aoc_ring_read_buffer(ptr, r, buffer, size); | |
| _region_advance_rx(r, bytes_read); | |
| } else { | |
| ptr = aoc_service_current_read_pointer(service, base, dir); | |
| hdr = (struct aoc_ipc_message_header *)ptr; | |
| ptr += sizeof(struct aoc_ipc_message_header); | |
| /* Validate length */ | |
| if (*size < hdr->length) | |
| return false; | |
| /* TODO: Length should always be little endian */ | |
| copy_from_buffer(buffer, ptr, hdr->length); | |
| *size = hdr->length; | |
| aoc_service_increment_read_index(service, dir); | |
| } | |
| return true; | |
| } | |
| bool aoc_service_write_message(aoc_service *service, void *base, | |
| aoc_direction dir, const void *buffer, | |
| size_t size) | |
| { | |
| struct aoc_ipc_service_header *s = | |
| (struct aoc_ipc_service_header *)service; | |
| struct aoc_ipc_memory_region *r; | |
| u8 *ptr; | |
| if (!service || !base || !buffer || !VALID_DIRECTION(dir)) | |
| return false; | |
| r = &s->regions[dir]; | |
| if (!aoc_service_can_write_message(service, dir)) | |
| return false; | |
| if (size > aoc_service_message_size(service, dir)) | |
| return false; | |
| if (aoc_service_is_ring(service)) { | |
| u32 bytes_written; | |
| ptr = aoc_service_ring_base(service, base, dir); | |
| bytes_written = _aoc_ring_write_buffer(ptr, r, buffer, size); | |
| WRITE_BARRIER(); | |
| _region_advance_tx(r, bytes_written); | |
| } else { | |
| ptr = aoc_service_current_write_pointer(service, base, dir); | |
| _aoc_queue_write_buffer(ptr, buffer, size); | |
| WRITE_BARRIER(); | |
| aoc_service_increment_write_index(service, dir); | |
| } | |
| return true; | |
| } | |
| void *aoc_service_ring_base(aoc_service *service, void *base, aoc_direction dir) | |
| { | |
| struct aoc_ipc_service_header *s = | |
| (struct aoc_ipc_service_header *)service; | |
| u32 size, offset; | |
| if (!service || !aoc_service_is_ring(service) || !VALID_DIRECTION(dir)) | |
| return NULL; | |
| size = s->regions[dir].size; | |
| offset = s->regions[dir].offset; | |
| return (size > 0) ? (void *)((uintptr_t)base + offset) : NULL; | |
| } | |
| size_t aoc_service_ring_size(aoc_service *service, aoc_direction dir) | |
| { | |
| struct aoc_ipc_service_header *s = | |
| (struct aoc_ipc_service_header *)service; | |
| if (!service || !aoc_service_is_ring(service) || !VALID_DIRECTION(dir)) | |
| return 0; | |
| return s->regions[dir].size; | |
| } | |
| size_t aoc_service_ring_read_offset(aoc_service *service, aoc_direction dir) | |
| { | |
| struct aoc_ipc_service_header *s = | |
| (struct aoc_ipc_service_header *)service; | |
| if (!service || !aoc_service_is_ring(service) || !VALID_DIRECTION(dir)) | |
| return 0; | |
| return ioread32(&s->regions[dir].rp); | |
| } | |
| size_t aoc_service_ring_write_offset(aoc_service *service, aoc_direction dir) | |
| { | |
| struct aoc_ipc_service_header *s = | |
| (struct aoc_ipc_service_header *)service; | |
| if (!service || !aoc_service_is_ring(service) || !VALID_DIRECTION(dir)) | |
| return 0; | |
| return ioread32(&(s->regions[dir].wp)); | |
| } | |
| bool aoc_ring_is_push(aoc_service *service) | |
| { | |
| struct aoc_ipc_service_header *s = | |
| (struct aoc_ipc_service_header *)service; | |
| if (!service || !aoc_service_is_ring(service)) | |
| return false; | |
| return (s->flags & AOC_SERVICE_FLAG_RING_PUSH) != 0; | |
| } |