| # Copyright 2016 The Gemmlowp Authors. All rights reserved. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| """32bit ARM/NEON assembly emitter. |
| |
| Used by code generators to produce ARM assembly with NEON simd code. |
| Provides tools for easier register management: named register variable |
| allocation/deallocation, and offers a more procedural/structured approach |
| to generating assembly. |
| |
| TODO: right now neon emitter prints out assembly instructions immediately, |
| it might be beneficial to keep the whole structure and emit the assembly after |
| applying some optimizations like: instruction reordering or register reuse. |
| |
| TODO: NeonRegister object assigns explicit registers at allocation time. |
| Similarily to emiting code, register mapping and reuse can be performed and |
| optimized lazily. |
| """ |
| |
| |
| class Error(Exception): |
| """Module level error.""" |
| |
| |
| class RegisterAllocationError(Error): |
| """Cannot alocate registers.""" |
| |
| |
| class LaneError(Error): |
| """Wrong lane number.""" |
| |
| |
| class ArgumentError(Error): |
| """Wrong argument.""" |
| |
| |
| def _Low(register): |
| assert register[0] == 'q' |
| num = int(register[1:]) |
| return 'd%d' % (num * 2) |
| |
| |
| def _High(register): |
| assert register[0] == 'q' |
| num = int(register[1:]) |
| return 'd%d' % (num * 2 + 1) |
| |
| |
| def _ExpandQuads(registers): |
| doubles = [] |
| for register in registers: |
| if register[0] == 'q': |
| doubles.append(_Low(register)) |
| doubles.append(_High(register)) |
| else: |
| doubles.append(register) |
| return doubles |
| |
| |
| def _MakeCompatible(op1, op2, op3): |
| if op1[0] == 'd' or op2[0] == 'd' or op3[0] == 'd': |
| if op1[0] == 'q': |
| op1 = _Low(op1) |
| if op2[0] == 'q': |
| op2 = _Low(op2) |
| if op3[0] == 'q': |
| op3 = _Low(op3) |
| return (op1, op2, op3) |
| |
| |
| class _NeonRegisters32Bit(object): |
| """Utility that keeps track of used 32bit ARM/NEON registers.""" |
| |
| def __init__(self): |
| self.double = set() |
| self.double_ever = set() |
| self.general = set() |
| self.general_ever = set() |
| self.parameters = dict() |
| self.output_parameters = dict() |
| |
| def MapParameter(self, parameter, parameter_value=None): |
| if not parameter_value: |
| parameter_value = parameter |
| self.parameters[parameter] = (parameter_value, 'r') |
| return '%%[%s]' % parameter |
| |
| def MapMemoryParameter(self, parameter, parameter_value=None): |
| if not parameter_value: |
| parameter_value = parameter |
| self.parameters[parameter] = (parameter_value, 'm') |
| return '%%[%s]' % parameter |
| |
| def MapOutputParameter(self, parameter, parameter_value=None): |
| if not parameter_value: |
| parameter_value = parameter |
| self.output_parameters[parameter] = (parameter_value, '+r') |
| return '%%[%s]' % parameter |
| |
| def DoubleRegister(self, min_val=0): |
| for i in range(min_val, 32): |
| if i not in self.double: |
| self.double.add(i) |
| self.double_ever.add(i) |
| return 'd%d' % i |
| raise RegisterAllocationError('Not enough double registers.') |
| |
| def QuadRegister(self, min_val=0): |
| for i in range(min_val, 16): |
| if ((i * 2) not in self.double) and ((i * 2 + 1) not in self.double): |
| self.double.add(i * 2) |
| self.double.add(i * 2 + 1) |
| self.double_ever.add(i * 2) |
| self.double_ever.add(i * 2 + 1) |
| return 'q%d' % i |
| raise RegisterAllocationError('Not enough quad registers.') |
| |
| def GeneralRegister(self): |
| for i in range(0, 16): |
| if i not in self.general: |
| self.general.add(i) |
| self.general_ever.add(i) |
| return 'r%d' % i |
| raise RegisterAllocationError('Not enough general registers.') |
| |
| def MappedParameters(self): |
| return [(k, v) for (k, v) in self.parameters.items()] |
| |
| def MappedOutputParameters(self): |
| return [(k, v) for (k, v) in self.output_parameters.items()] |
| |
| def Clobbers(self): |
| return (['r%d' % i for i in self.general_ever] + |
| ['d%d' % i for i in self.DoubleClobbers()]) |
| |
| def DoubleClobbers(self): |
| return sorted(self.double_ever) |
| |
| def FreeRegister(self, register): |
| assert len(register) > 1 |
| if register[0] not in ['r', 'd', 'q']: |
| return |
| |
| num = int(register[1:]) |
| |
| if register[0] == 'r': |
| assert num in self.general |
| self.general.remove(num) |
| elif register[0] == 'd': |
| assert num in self.double |
| self.double.remove(num) |
| elif register[0] == 'q': |
| assert num * 2 in self.double |
| assert num * 2 + 1 in self.double |
| self.double.remove(num * 2) |
| self.double.remove(num * 2 + 1) |
| else: |
| raise RegisterDeallocationError('Register not allocated: %s' % register) |
| |
| def FreeRegisters(self, registers): |
| for register in registers: |
| self.FreeRegister(register) |
| |
| |
| class NeonEmitter(object): |
| """Emits ARM/NEON assembly opcodes.""" |
| |
| def __init__(self, debug=False): |
| self.ops = {} |
| self.indent = '' |
| self.debug = debug |
| |
| def PushIndent(self, delta=' '): |
| self.indent += delta |
| |
| def PopIndent(self, delta=2): |
| self.indent = self.indent[:-delta] |
| |
| def EmitIndented(self, what): |
| print self.indent + what |
| |
| def PushOp(self, op): |
| if op in self.ops.keys(): |
| self.ops[op] += 1 |
| else: |
| self.ops[op] = 1 |
| |
| def ClearCounters(self): |
| self.ops.clear() |
| |
| def EmitNewline(self): |
| print '' |
| |
| def EmitPreprocessor1(self, op, param): |
| print '#%s %s' % (op, param) |
| |
| def EmitPreprocessor(self, op): |
| print '#%s' % op |
| |
| def EmitInclude(self, include): |
| self.EmitPreprocessor1('include', include) |
| |
| def EmitCall1(self, function, param): |
| self.EmitIndented('%s(%s);' % (function, param)) |
| |
| def EmitAssert(self, assert_expression): |
| if self.debug: |
| self.EmitCall1('assert', assert_expression) |
| |
| def EmitHeaderBegin(self, header_name, includes): |
| self.EmitPreprocessor1('ifndef', (header_name + '_H_').upper()) |
| self.EmitPreprocessor1('define', (header_name + '_H_').upper()) |
| self.EmitNewline() |
| if includes: |
| for include in includes: |
| self.EmitInclude(include) |
| self.EmitNewline() |
| |
| def EmitHeaderEnd(self): |
| self.EmitPreprocessor('endif') |
| |
| def EmitCode(self, code): |
| self.EmitIndented('%s;' % code) |
| |
| def EmitFunctionBeginA(self, function_name, params, return_type): |
| self.EmitIndented('%s %s(%s) {' % |
| (return_type, function_name, |
| ', '.join(['%s %s' % (t, n) for (t, n) in params]))) |
| self.PushIndent() |
| |
| def EmitFunctionEnd(self): |
| self.PopIndent() |
| self.EmitIndented('}') |
| |
| def EmitAsmBegin(self): |
| self.EmitIndented('asm volatile(') |
| self.PushIndent() |
| |
| def EmitAsmMapping(self, elements): |
| if elements: |
| self.EmitIndented(': ' + ', '.join( |
| ['[%s] "%s"(%s)' % (d, v[1], v[0]) for (d, v) in elements])) |
| else: |
| self.EmitIndented(':') |
| |
| def EmitClobbers(self, elements): |
| if elements: |
| self.EmitIndented(': ' + ', '.join(['"%s"' % c for c in elements])) |
| else: |
| self.EmitIndented(':') |
| |
| def EmitAsmEnd(self, registers): |
| self.EmitAsmMapping(registers.MappedOutputParameters()) |
| self.EmitAsmMapping(registers.MappedParameters()) |
| self.EmitClobbers(registers.Clobbers() + ['cc', 'memory']) |
| self.PopIndent() |
| self.EmitIndented(');') |
| |
| def EmitComment(self, comment): |
| self.EmitIndented('// ' + comment) |
| |
| def EmitNumericalLabel(self, label): |
| self.EmitIndented('"%d:"' % label) |
| |
| def EmitOp1(self, op, param1): |
| self.PushOp(op) |
| self.EmitIndented('"%s %s\\n"' % (op, param1)) |
| |
| def EmitOp2(self, op, param1, param2): |
| self.PushOp(op) |
| self.EmitIndented('"%s %s, %s\\n"' % (op, param1, param2)) |
| |
| def EmitOp3(self, op, param1, param2, param3): |
| self.PushOp(op) |
| self.EmitIndented('"%s %s, %s, %s\\n"' % (op, param1, param2, param3)) |
| |
| def EmitAdd(self, destination, source, param): |
| self.EmitOp3('add', destination, source, param) |
| |
| def EmitSubs(self, destination, source, param): |
| self.EmitOp3('subs', destination, source, param) |
| |
| def EmitSub(self, destination, source, param): |
| self.EmitOp3('sub', destination, source, param) |
| |
| def EmitMul(self, destination, source, param): |
| self.EmitOp3('mul', destination, source, param) |
| |
| def EmitMov(self, param1, param2): |
| self.EmitOp2('mov', param1, param2) |
| |
| def EmitBeqBack(self, label): |
| self.EmitOp1('beq', '%db' % label) |
| |
| def EmitBeqFront(self, label): |
| self.EmitOp1('beq', '%df' % label) |
| |
| def EmitBgtBack(self, label): |
| self.EmitOp1('bgt', '%db' % label) |
| |
| def EmitBgtFront(self, label): |
| self.EmitOp1('bgt', '%df' % label) |
| |
| def EmitBleBack(self, label): |
| self.EmitOp1('ble', '%db' % label) |
| |
| def EmitBleFront(self, label): |
| self.EmitOp1('ble', '%df' % label) |
| |
| def EmitBneBack(self, label): |
| self.EmitOp1('bne', '%db' % label) |
| |
| def EmitBneFront(self, label): |
| self.EmitOp1('bne', '%df' % label) |
| |
| def EmitVAdd(self, add_type, destination, source_1, source_2): |
| destination, source_1, source_2 = _MakeCompatible(destination, source_1, |
| source_2) |
| self.EmitOp3('vadd.%s' % add_type, destination, source_1, source_2) |
| |
| def EmitVAddw(self, add_type, destination, source_1, source_2): |
| self.EmitOp3('vaddw.%s' % add_type, destination, source_1, source_2) |
| |
| def EmitVSub(self, sub_type, destination, source_1, source_2): |
| destination, source_1, source_2 = _MakeCompatible(destination, source_1, |
| source_2) |
| self.EmitOp3('vsub.%s' % sub_type, destination, source_1, source_2) |
| |
| def EmitVCvt(self, cvt_to, cvt_from, destination, source): |
| self.EmitOp2('vcvt.%s.%s' % (cvt_to, cvt_from), destination, source) |
| |
| def EmitVDup(self, dup_type, destination, source): |
| self.EmitOp2('vdup.%s' % dup_type, destination, source) |
| |
| def EmitVMax(self, size, destination, source_1, source_2): |
| self.EmitOp3('vmax.%s' % size, destination, source_1, source_2) |
| |
| def EmitVMin(self, size, destination, source_1, source_2): |
| self.EmitOp3('vmin.%s' % size, destination, source_1, source_2) |
| |
| def EmitVMov(self, mov_type, destination, source): |
| self.EmitOp2('vmov.%s' % mov_type, destination, source) |
| |
| def EmitVMovl(self, mov_type, destination, source): |
| if source[0] == 'q': |
| source = _Low(source) |
| self.EmitOp2('vmovl.%s' % mov_type, destination, source) |
| |
| def EmitVMovl2(self, mov_type, destination_1, destination_2, source): |
| self.EmitVMovl(mov_type, destination_2, _High(source)) |
| self.EmitVMovl(mov_type, destination_1, _Low(source)) |
| |
| def EmitVQmovn(self, mov_type, destination, source): |
| if destination[0] == 'q': |
| destination = _Low(destination) |
| self.EmitOp2('vqmovn.%s' % mov_type, destination, source) |
| |
| def EmitVQmovn2(self, mov_type, destination, source_1, source_2): |
| self.EmitVQmovn(mov_type, _Low(destination), source_1) |
| self.EmitVQmovn(mov_type, _High(destination), source_2) |
| |
| def EmitVQmovun(self, mov_type, destination, source): |
| if destination[0] == 'q': |
| destination = _Low(destination) |
| self.EmitOp2('vqmovun.%s' % mov_type, destination, source) |
| |
| def EmitVQmovun2(self, mov_type, destination, source_1, source_2): |
| self.EmitVQmovun(mov_type, _Low(destination), source_1) |
| self.EmitVQmovun(mov_type, _High(destination), source_2) |
| |
| def EmitVMul(self, mul_type, destination, source_1, source_2): |
| destination, source_1, source_2 = _MakeCompatible(destination, source_1, |
| source_2) |
| self.EmitOp3('vmul.%s' % mul_type, destination, source_1, source_2) |
| |
| def EmitVMulScalar(self, mul_type, destination, source_1, source_2): |
| self.EmitOp3('vmul.%s' % mul_type, destination, source_1, source_2) |
| |
| def EmitVMull(self, mul_type, destination, source_1, source_2): |
| self.EmitOp3('vmull.%s' % mul_type, destination, source_1, source_2) |
| |
| def EmitVPadd(self, add_type, destination, source_1, source_2): |
| self.EmitOp3('vpadd.%s' % add_type, destination, source_1, source_2) |
| |
| def EmitVPaddl(self, add_type, destination, source): |
| self.EmitOp2('vpaddl.%s' % add_type, destination, source) |
| |
| def EmitVPadal(self, add_type, destination, source): |
| self.EmitOp2('vpadal.%s' % add_type, destination, source) |
| |
| def EmitLdr(self, register, value): |
| self.EmitOp2('ldr', register, value) |
| |
| def EmitVLoad(self, load_no, load_type, destination, source): |
| self.EmitVLoadA(load_no, load_type, [destination], source) |
| |
| def EmitVLoadA(self, load_no, load_type, destinations, source): |
| self.EmitOp2('vld%d.%d' % (load_no, load_type), |
| '{%s}' % ', '.join(_ExpandQuads(destinations)), source) |
| |
| def EmitVLoadAE(self, |
| load_type, |
| elem_count, |
| destinations, |
| source, |
| alignment=None): |
| bits_to_load = load_type * elem_count |
| destinations = _ExpandQuads(destinations) |
| if len(destinations) * 64 < bits_to_load: |
| raise ArgumentError('To few destinations: %d to load %d bits.' % |
| (len(destinations), bits_to_load)) |
| |
| while bits_to_load > 0: |
| if bits_to_load >= 256: |
| self.EmitVLoadA(1, 32, destinations[:4], |
| self.DereferenceIncrement(source, alignment)) |
| bits_to_load -= 256 |
| destinations = destinations[4:] |
| elif bits_to_load >= 192: |
| self.EmitVLoadA(1, 32, destinations[:3], |
| self.DereferenceIncrement(source, alignment)) |
| bits_to_load -= 192 |
| destinations = destinations[3:] |
| elif bits_to_load >= 128: |
| self.EmitVLoadA(1, 32, destinations[:2], |
| self.DereferenceIncrement(source, alignment)) |
| bits_to_load -= 128 |
| destinations = destinations[2:] |
| elif bits_to_load >= 64: |
| self.EmitVLoad(1, 32, destinations[0], |
| self.DereferenceIncrement(source, alignment)) |
| bits_to_load -= 64 |
| destinations = destinations[1:] |
| else: |
| destination = destinations[0] |
| if bits_to_load == 56: |
| self.EmitVLoad(1, 32, |
| self.Lane(32, destination, 0), |
| self.DereferenceIncrement(source)) |
| self.EmitVLoad(1, 16, |
| self.Lane(16, destination, 2), |
| self.DereferenceIncrement(source)) |
| self.EmitVLoad(1, 8, |
| self.Lane(8, destination, 6), |
| self.DereferenceIncrement(source)) |
| elif bits_to_load == 48: |
| self.EmitVLoad(1, 32, |
| self.Lane(32, destination, 0), |
| self.DereferenceIncrement(source)) |
| self.EmitVLoad(1, 16, |
| self.Lane(16, destination, 2), |
| self.DereferenceIncrement(source)) |
| elif bits_to_load == 40: |
| self.EmitVLoad(1, 32, |
| self.Lane(32, destination, 0), |
| self.DereferenceIncrement(source)) |
| self.EmitVLoad(1, 8, |
| self.Lane(8, destination, 4), |
| self.DereferenceIncrement(source)) |
| elif bits_to_load == 32: |
| self.EmitVLoad(1, 32, |
| self.Lane(32, destination, 0), |
| self.DereferenceIncrement(source)) |
| elif bits_to_load == 24: |
| self.EmitVLoad(1, 16, |
| self.Lane(16, destination, 0), |
| self.DereferenceIncrement(source)) |
| self.EmitVLoad(1, 8, |
| self.Lane(8, destination, 2), |
| self.DereferenceIncrement(source)) |
| elif bits_to_load == 16: |
| self.EmitVLoad(1, 16, |
| self.Lane(16, destination, 0), |
| self.DereferenceIncrement(source)) |
| elif bits_to_load == 8: |
| self.EmitVLoad(1, 8, |
| self.Lane(8, destination, 0), |
| self.DereferenceIncrement(source)) |
| else: |
| raise ArgumentError('Wrong leftover: %d' % bits_to_load) |
| return |
| |
| def EmitVLoadE(self, load_type, count, destination, source, alignment=None): |
| self.EmitVLoadAE(load_type, count, [destination], source, alignment) |
| |
| def EmitVLoadAllLanes(self, load_no, load_type, destination, source): |
| destinations = [] |
| if destination[0] == 'q': |
| destinations.append(self.AllLanes(_Low(destination))) |
| destinations.append(self.AllLanes(_High(destination))) |
| else: |
| destinations.append(self.AllLanes(destination)) |
| self.EmitVLoadA(load_no, load_type, destinations, source) |
| |
| def EmitVLoadOffset(self, load_no, load_type, destination, source, offset): |
| self.EmitVLoadOffsetA(load_no, load_type, [destination], source, offset) |
| |
| def EmitVLoadOffsetA(self, load_no, load_type, destinations, source, offset): |
| assert len(destinations) <= 4 |
| self.EmitOp3('vld%d.%d' % (load_no, load_type), |
| '{%s}' % ', '.join(_ExpandQuads(destinations)), source, offset) |
| |
| def EmitPld(self, load_address_register): |
| self.EmitOp1('pld', '[%s]' % load_address_register) |
| |
| def EmitPldw(self, store_address_register): |
| self.EmitOp1('pldw', '[%s]' % store_address_register) |
| |
| def EmitPldOffset(self, load_address_register, offset): |
| self.EmitOp1('pld', '[%s, %s]' % (load_address_register, offset)) |
| |
| def EmitPldwOffset(self, store_address_register, offset): |
| self.EmitOp1('pldw', '[%s, %s]' % (store_address_register, offset)) |
| |
| def EmitVShl(self, shift_type, destination, source, shift): |
| self.EmitOp3('vshl.%s' % shift_type, destination, source, shift) |
| |
| def EmitVStore(self, store_no, store_type, source, destination): |
| self.EmitVStoreA(store_no, store_type, [source], destination) |
| |
| def EmitVStoreA(self, store_no, store_type, sources, destination): |
| self.EmitOp2('vst%d.%d' % (store_no, store_type), |
| '{%s}' % ', '.join(_ExpandQuads(sources)), destination) |
| |
| def EmitVStoreAE(self, |
| store_type, |
| elem_count, |
| sources, |
| destination, |
| alignment=None): |
| bits_to_store = store_type * elem_count |
| sources = _ExpandQuads(sources) |
| if len(sources) * 64 < bits_to_store: |
| raise ArgumentError('To few sources: %d to store %d bits.' % |
| (len(sources), bits_to_store)) |
| |
| while bits_to_store > 0: |
| if bits_to_store >= 256: |
| self.EmitVStoreA(1, 32, sources[:4], |
| self.DereferenceIncrement(destination, alignment)) |
| bits_to_store -= 256 |
| sources = sources[4:] |
| elif bits_to_store >= 192: |
| self.EmitVStoreA(1, 32, sources[:3], |
| self.DereferenceIncrement(destination, alignment)) |
| bits_to_store -= 192 |
| sources = sources[3:] |
| elif bits_to_store >= 128: |
| self.EmitVStoreA(1, 32, sources[:2], |
| self.DereferenceIncrement(destination, alignment)) |
| bits_to_store -= 128 |
| sources = sources[2:] |
| elif bits_to_store >= 64: |
| self.EmitVStore(1, 32, sources[0], |
| self.DereferenceIncrement(destination, alignment)) |
| bits_to_store -= 64 |
| sources = sources[1:] |
| else: |
| source = sources[0] |
| if bits_to_store == 56: |
| self.EmitVStore(1, 32, |
| self.Lane(32, source, 0), |
| self.DereferenceIncrement(destination)) |
| self.EmitVStore(1, 16, |
| self.Lane(16, source, 2), |
| self.DereferenceIncrement(destination)) |
| self.EmitVStore(1, 8, |
| self.Lane(8, source, 6), |
| self.DereferenceIncrement(destination)) |
| elif bits_to_store == 48: |
| self.EmitVStore(1, 32, |
| self.Lane(32, source, 0), |
| self.DereferenceIncrement(destination)) |
| self.EmitVStore(1, 16, |
| self.Lane(16, source, 2), |
| self.DereferenceIncrement(destination)) |
| elif bits_to_store == 40: |
| self.EmitVStore(1, 32, |
| self.Lane(32, source, 0), |
| self.DereferenceIncrement(destination)) |
| self.EmitVStore(1, 8, |
| self.Lane(8, source, 4), |
| self.DereferenceIncrement(destination)) |
| elif bits_to_store == 32: |
| self.EmitVStore(1, 32, |
| self.Lane(32, source, 0), |
| self.DereferenceIncrement(destination)) |
| elif bits_to_store == 24: |
| self.EmitVStore(1, 16, |
| self.Lane(16, source, 0), |
| self.DereferenceIncrement(destination)) |
| self.EmitVStore(1, 8, |
| self.Lane(8, source, 2), |
| self.DereferenceIncrement(destination)) |
| elif bits_to_store == 16: |
| self.EmitVStore(1, 16, |
| self.Lane(16, source, 0), |
| self.DereferenceIncrement(destination)) |
| elif bits_to_store == 8: |
| self.EmitVStore(1, 8, |
| self.Lane(8, source, 0), |
| self.DereferenceIncrement(destination)) |
| else: |
| raise ArgumentError('Wrong leftover: %d' % bits_to_store) |
| return |
| |
| def EmitVStoreE(self, store_type, count, source, destination, alignment=None): |
| self.EmitVStoreAE(store_type, count, [source], destination, alignment) |
| |
| def EmitVStoreOffset(self, store_no, store_type, source, destination, offset): |
| self.EmitVStoreOffsetA(store_no, store_type, [source], destination, offset) |
| |
| def EmitVStoreOffsetA(self, store_no, store_type, sources, destination, |
| offset): |
| self.EmitOp3('vst%d.%d' % (store_no, store_type), |
| '{%s}' % ', '.join(_ExpandQuads(sources)), destination, offset) |
| |
| def EmitVStoreOffsetE(self, store_type, count, source, destination, offset): |
| """Emit assembly to store a number elements from the source registers.""" |
| if store_type is not 32: |
| raise ArgumentError('Unsupported store_type: %d' % store_type) |
| |
| sources = [] |
| if source[0] == 'q': |
| sources.append(_Low(source)) |
| sources.append(_High(source)) |
| if count * store_type > 128: |
| raise ArgumentError('To many %dbit elements in a q register: %d' % |
| (store_type, count)) |
| else: |
| sources.append(source) |
| if count * store_type > 64: |
| raise ArgumentError('To many %dbit elements in a d register: %d' % |
| (store_type, count)) |
| |
| if count == 1: |
| self.EmitVStoreOffset(1, store_type, |
| self.Lane(store_type, sources[0], 0), |
| self.Dereference(destination, None), offset) |
| elif count == 2: |
| self.EmitVStoreOffset(1, store_type, sources[0], |
| self.Dereference(destination, None), offset) |
| elif count == 3: |
| self.EmitVStore(1, store_type, sources[0], |
| self.DereferenceIncrement(destination, None)) |
| self.EmitVStoreOffset(1, store_type, |
| self.Lane(store_type, sources[1], 0), |
| self.Dereference(destination, None), offset) |
| self.EmitSub(destination, destination, self.ImmediateConstant(8)) |
| elif count == 4: |
| self.EmitVStoreOffsetA(1, store_type, sources, |
| self.Dereference(destination, None), offset) |
| else: |
| raise ArgumentError('To many elements: %d' % count) |
| |
| def EmitVSumReduce(self, reduce_type, elem_count, reduce_count, destinations, |
| sources): |
| """Emit assembly for n-fold horizontal sum reduction.""" |
| if reduce_type is not 'u32': |
| raise ArgumentError('Unsupported reduce: %s' % reduce_type) |
| |
| sources = _ExpandQuads(sources) |
| |
| destinations = _ExpandQuads(destinations) |
| |
| if len(destinations) * 2 < elem_count: |
| raise ArgumentError('Not enough space in destination: %d vs %d' % |
| (len(destinations) * 2, elem_count)) |
| |
| if len(sources) * 2 != elem_count * reduce_count: |
| raise ArgumentError('Wrong number of sources: %d vs %d' % |
| (len(sources) * 2, elem_count * reduce_count)) |
| |
| if reduce_count <= 1: |
| raise ArgumentError('Unsupported reduce_count: %d' % reduce_count) |
| |
| while reduce_count > 1: |
| if len(sources) % 2 == 1: |
| sources.append(sources[-1]) |
| |
| if reduce_count == 2: |
| for i in range(len(sources) / 2): |
| self.EmitVPadd(reduce_type, destinations[i], sources[2 * i], |
| sources[2 * i + 1]) |
| return |
| else: |
| sources_2 = [] |
| for i in range(len(sources) / 2): |
| self.EmitVPadd(reduce_type, sources[2 * i], sources[2 * i], |
| sources[2 * i + 1]) |
| sources_2.append(sources[2 * i]) |
| reduce_count /= 2 |
| sources = sources_2 |
| |
| def EmitVUzp(self, uzp_type, operand_1, operand_2): |
| self.EmitOp2('vuzp.%d' % uzp_type, operand_1, operand_2) |
| |
| def EmitVTrn(self, trn_type, operand_1, operand_2): |
| self.EmitOp2('vtrn.%d' % trn_type, operand_1, operand_2) |
| |
| def EmitColBlockStride(self, cols, stride, new_stride): |
| assert cols in [1, 2, 3, 4, 5, 6, 7, 8] |
| if cols in [5, 6, 7]: |
| self.EmitSub(new_stride, stride, self.ImmediateConstant(4)) |
| |
| def EmitLoadColBlock(self, unused_registers, load_type, cols, elements, block, |
| input_address, stride): |
| """Load a block of column major data.""" |
| assert cols is len(block) |
| assert load_type is 8 |
| |
| input_deref = self.Dereference(input_address, None) |
| input_deref_increment = self.DereferenceIncrement(input_address, None) |
| |
| if cols is 1: |
| for i in range(elements): |
| self.EmitVLoadOffset(1, 8, |
| self.Lane(8, block[0], i), input_deref, stride) |
| self.EmitPld(input_address) |
| elif cols is 2: |
| for i in range(elements): |
| self.EmitVLoadOffset(1, 16, |
| self.Lane(16, block[i / 4], i % 4), input_deref, |
| stride) |
| self.EmitPld(input_address) |
| self.EmitVUzp(8, block[0], block[1]) |
| elif cols is 3: |
| for i in range(elements): |
| self.EmitVLoadOffsetA(3, 8, [self.Lane(8, row, i) for row in block], |
| input_deref, stride) |
| elif cols is 4: |
| for i in range(elements): |
| self.EmitVLoadOffset(1, 32, |
| self.Lane(32, block[i % 4], i / 4), input_deref, |
| stride) |
| self.EmitPld(input_address) |
| self.EmitVTrn(16, block[0], block[2]) |
| self.EmitVTrn(16, block[1], block[3]) |
| self.EmitVTrn(8, block[0], block[1]) |
| self.EmitVTrn(8, block[2], block[3]) |
| elif cols is 5: |
| for i in range(elements): |
| self.EmitVLoad(1, 32, |
| self.Lane(32, block[i % 4], i / 4), |
| input_deref_increment) |
| self.EmitVLoadOffset(1, 8, |
| self.Lane(8, block[4], i), input_deref, stride) |
| self.EmitPld(input_address) |
| self.EmitVTrn(16, block[0], block[2]) |
| self.EmitVTrn(16, block[1], block[3]) |
| self.EmitVTrn(8, block[0], block[1]) |
| self.EmitVTrn(8, block[2], block[3]) |
| elif cols is 6: |
| for i in range(elements): |
| self.EmitVLoad(1, 32, |
| self.Lane(32, block[i % 4], i / 4), |
| input_deref_increment) |
| self.EmitVLoadOffset(1, 16, |
| self.Lane(16, block[4 + i / 4], i % 4), |
| input_deref, stride) |
| self.EmitPld(input_address) |
| self.EmitVTrn(16, block[0], block[2]) |
| self.EmitVTrn(16, block[1], block[3]) |
| self.EmitVUzp(8, block[4], block[5]) |
| self.EmitVTrn(8, block[0], block[1]) |
| self.EmitVTrn(8, block[2], block[3]) |
| elif cols is 7: |
| for i in range(elements): |
| self.EmitVLoad(1, 32, |
| self.Lane(32, block[i % 4], i / 4), |
| input_deref_increment) |
| self.EmitVLoadOffsetA(3, 8, |
| [self.Lane(8, row, i) for row in block[4:]], |
| input_deref, stride) |
| self.EmitPld(input_address) |
| self.EmitVTrn(16, block[0], block[2]) |
| self.EmitVTrn(16, block[1], block[3]) |
| self.EmitVTrn(8, block[0], block[1]) |
| self.EmitVTrn(8, block[2], block[3]) |
| elif cols is 8: |
| for i in range(elements): |
| self.EmitVLoadOffset(1, 32, block[i], input_deref, stride) |
| self.EmitPld(input_address) |
| self.EmitVTrn(8, block[0], block[1]) |
| self.EmitVTrn(8, block[2], block[3]) |
| self.EmitVTrn(8, block[4], block[5]) |
| self.EmitVTrn(8, block[6], block[7]) |
| self.EmitVTrn(16, block[0], block[2]) |
| self.EmitVTrn(16, block[1], block[3]) |
| self.EmitVTrn(16, block[4], block[6]) |
| self.EmitVTrn(16, block[5], block[7]) |
| self.EmitVTrn(32, block[0], block[4]) |
| self.EmitVTrn(32, block[1], block[5]) |
| self.EmitVTrn(32, block[2], block[6]) |
| self.EmitVTrn(32, block[3], block[7]) |
| else: |
| assert False |
| return block |
| |
| def Dereference(self, value, alignment=None): |
| if alignment: |
| return '[%s:%d]' % (value, alignment) |
| else: |
| return '[%s]' % value |
| |
| def DereferenceIncrement(self, value, alignment=None): |
| return '%s!' % self.Dereference(value, alignment) |
| |
| def ImmediateConstant(self, value): |
| return '#%d' % value |
| |
| def AllLanes(self, value): |
| return '%s[]' % value |
| |
| def Lane(self, bits, value, lane): |
| """Get the proper n-bit lane from the given register.""" |
| registers = [] |
| if value[0] == 'q': |
| registers.append(_Low(value)) |
| registers.append(_High(value)) |
| else: |
| registers.append(value) |
| |
| elems_per_register = 64 / bits |
| register = lane / elems_per_register |
| lane %= elems_per_register |
| |
| return '%s[%d]' % (registers[register], lane) |
| |
| def CreateRegisters(self): |
| return _NeonRegisters32Bit() |
