pw_web: Fix printf decoder to work with varints
Change-Id: I039242331ba7253333c6374c2d1854ac33e921da
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/129588
Commit-Queue: Asad Memon <asadmemon@google.com>
Reviewed-by: Alexei Frolov <frolv@google.com>
Reviewed-by: Wyatt Hepler <hepler@google.com>
diff --git a/package-lock.json b/package-lock.json
index fbc143a..d00b93c 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,16 +1,17 @@
{
"name": "pigweedjs",
- "version": "0.0.5",
+ "version": "0.0.7",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "pigweedjs",
- "version": "0.0.5",
+ "version": "0.0.7",
"license": "Apache-2.0",
"dependencies": {
"@protobuf-ts/protoc": "^2.7.0",
"google-protobuf": "^3.17.3",
+ "long": "^5.2.1",
"object-path": "^0.11.8",
"ts-protoc-gen": "^0.15.0"
},
@@ -784,6 +785,12 @@
"node": ">=6"
}
},
+ "node_modules/@grpc/proto-loader/node_modules/long": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz",
+ "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==",
+ "dev": true
+ },
"node_modules/@grpc/proto-loader/node_modules/protobufjs": {
"version": "6.11.2",
"dev": true,
@@ -7032,9 +7039,9 @@
"license": "MIT"
},
"node_modules/long": {
- "version": "4.0.0",
- "dev": true,
- "license": "Apache-2.0"
+ "version": "5.2.1",
+ "resolved": "https://registry.npmjs.org/long/-/long-5.2.1.tgz",
+ "integrity": "sha512-GKSNGeNAtw8IryjjkhZxuKB3JzlcLTwjtiQCHKvqQet81I93kXslhDQruGI/QsddO83mcDToBVy7GqGS/zYf/A=="
},
"node_modules/loose-envify": {
"version": "1.4.0",
@@ -10604,6 +10611,12 @@
"yargs": "^16.1.1"
},
"dependencies": {
+ "long": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz",
+ "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==",
+ "dev": true
+ },
"protobufjs": {
"version": "6.11.2",
"dev": true,
@@ -15068,8 +15081,9 @@
"dev": true
},
"long": {
- "version": "4.0.0",
- "dev": true
+ "version": "5.2.1",
+ "resolved": "https://registry.npmjs.org/long/-/long-5.2.1.tgz",
+ "integrity": "sha512-GKSNGeNAtw8IryjjkhZxuKB3JzlcLTwjtiQCHKvqQet81I93kXslhDQruGI/QsddO83mcDToBVy7GqGS/zYf/A=="
},
"loose-envify": {
"version": "1.4.0",
diff --git a/package.json b/package.json
index 837da7f..dc85bc9 100644
--- a/package.json
+++ b/package.json
@@ -75,6 +75,7 @@
"dependencies": {
"@protobuf-ts/protoc": "^2.7.0",
"google-protobuf": "^3.17.3",
+ "long": "^5.2.1",
"object-path": "^0.11.8",
"ts-protoc-gen": "^0.15.0"
},
diff --git a/pw_tokenizer/ts/detokenizer.ts b/pw_tokenizer/ts/detokenizer.ts
index 90bff2b..fe6ea91 100644
--- a/pw_tokenizer/ts/detokenizer.ts
+++ b/pw_tokenizer/ts/detokenizer.ts
@@ -115,7 +115,7 @@
data.byteOffset,
4
).getUint32(0, true);
- const args = new Uint8Array(data.buffer.slice(4));
+ const args = new Uint8Array(data.buffer.slice(data.byteOffset + 4));
return {token, args};
}
diff --git a/pw_tokenizer/ts/int_testdata.ts b/pw_tokenizer/ts/int_testdata.ts
new file mode 100644
index 0000000..36b5a21
--- /dev/null
+++ b/pw_tokenizer/ts/int_testdata.ts
@@ -0,0 +1,52 @@
+// Copyright 2023 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+const IntDB = [
+ ["%d", "-128", "%u", "4294967168", '\xff\x01'],
+ ["%d", "-10", "%u", "4294967286", '\x13'],
+ ["%d", "-9", "%u", "4294967287", '\x11'],
+ ["%d", "-8", "%u", "4294967288", '\x0f'],
+ ["%d", "-7", "%u", "4294967289", '\x0d'],
+ ["%d", "-6", "%u", "4294967290", '\x0b'],
+ ["%d", "-5", "%u", "4294967291", '\x09'],
+ ["%d", "-4", "%u", "4294967292", '\x07'],
+ ["%d", "-3", "%u", "4294967293", '\x05'],
+ ["%d", "-2", "%u", "4294967294", '\x03'],
+ ["%d", "-1", "%u", "4294967295", '\x01'],
+ ["%d", "0", "%u", "0", '\x00'],
+ ["%d", "1", "%u", "1", '\x02'],
+ ["%d", "2", "%u", "2", '\x04'],
+ ["%d", "3", "%u", "3", '\x06'],
+ ["%d", "4", "%u", "4", '\x08'],
+ ["%d", "5", "%u", "5", '\x0a'],
+ ["%d", "6", "%u", "6", '\x0c'],
+ ["%d", "7", "%u", "7", '\x0e'],
+ ["%d", "8", "%u", "8", '\x10'],
+ ["%d", "9", "%u", "9", '\x12'],
+ ["%d", "10", "%u", "10", '\x14'],
+ ["%d", "127", "%u", "127", '\xfe\x01'],
+ ["%d", "-32768", "%u", "4294934528", '\xff\xff\x03'],
+ ["%d", "652344632", "%u", "652344632", '\xf0\xf4\x8f\xee\x04'],
+ ["%d", "18567", "%u", "18567", '\x8e\xa2\x02'],
+ ["%d", "-14", "%u", "4294967282", '\x1b'],
+ ["%d", "-2147483648", "%u", "2147483648", '\xff\xff\xff\xff\x0f'],
+ ["%ld", "-14", "%lu", "4294967282", '\x1b'],
+ ["%d", "2075650855", "%u", "2075650855", '\xce\xac\xbf\xbb\x0f'],
+ ["%lld", "5922204476835468009", "%llu", "5922204476835468009", '\xd2\xcb\x8c\x90\x86\xe6\xf2\xaf\xa4\x01'],
+ ["%lld", "-9223372036854775808", "%llu", "9223372036854775808", '\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01'],
+ ["%lld", "3273441488341945355", "%llu", "3273441488341945355", '\x96\xb0\xae\x9a\x96\xec\xcc\xed\x5a'],
+ ["%lld", "-9223372036854775807", "%llu", "9223372036854775809", '\xfd\xff\xff\xff\xff\xff\xff\xff\xff\x01'],
+]
+
+export default IntDB;
diff --git a/pw_tokenizer/ts/printf_decoder.ts b/pw_tokenizer/ts/printf_decoder.ts
index 127d358..7b02030 100644
--- a/pw_tokenizer/ts/printf_decoder.ts
+++ b/pw_tokenizer/ts/printf_decoder.ts
@@ -13,9 +13,9 @@
// the License.
/** Decodes arguments and formats them with the provided format string. */
+import Long from "long";
-const SPECIFIER_REGEX = /%(\.([0-9]+))?([%csdioxXufFeEaAgGnp])/g;
-
+const SPECIFIER_REGEX = /%(\.([0-9]+))?(hh|h|ll|l|j|z|t|L)?([%csdioxXufFeEaAgGnp])/g;
// Conversion specifiers by type; n is not supported.
const SIGNED_INT = 'di'.split('');
const UNSIGNED_INT = 'oxXup'.split('');
@@ -33,14 +33,24 @@
interface DecodedArg {
size: number;
- value: string | number | null;
+ value: string | number | Long | null;
}
// ZigZag decode function from protobuf's wire_format module.
-function zigzagDecode(value: number): number {
- if (!(value & 0x1)) return value >> 1;
- return (value >> 1) ^ ~0;
-}
+function zigzagDecode(value: Long, unsigned: boolean = false): Long {
+ // 64 bit math is:
+ // signmask = (zigzag & 1) ? -1 : 0;
+ // twosComplement = (zigzag >> 1) ^ signmask;
+ //
+ // To work with 32 bit, we can operate on both but "carry" the lowest bit
+ // from the high word by shifting it up 31 bits to be the most significant bit
+ // of the low word.
+ var bitsLow = value.low, bitsHigh = value.high;
+ var signFlipMask = -(bitsLow & 1);
+ bitsLow = ((bitsLow >>> 1) | (bitsHigh << 31)) ^ signFlipMask;
+ bitsHigh = (bitsHigh >>> 1) ^ signFlipMask;
+ return new Long(bitsLow, bitsHigh, unsigned);
+};
export class PrintfDecoder {
// Reads a unicode string from the encoded data.
@@ -65,16 +75,19 @@
}
private decodeSignedInt(args: Uint8Array): DecodedArg {
- if (args.length === 0) return {size: 0, value: null};
+ return this._decodeInt(args);
+ }
+ private _decodeInt(args: Uint8Array, unsigned: boolean = false): DecodedArg {
+ if (args.length === 0) return {size: 0, value: null};
let count = 0;
- let result = 0;
+ let result = new Long(0);
let shift = 0;
for (count = 0; count < args.length; count++) {
const byte = args[count];
- result |= (byte & 0x7f) << shift;
+ result = result.or((Long.fromInt(byte, unsigned).and(0x7f)).shiftLeft(shift));
if (!(byte & 0x80)) {
- return {value: zigzagDecode(result), size: count + 1};
+ return {value: zigzagDecode(result, unsigned), size: count + 1};
}
shift += 7;
if (shift >= 64) break;
@@ -83,15 +96,21 @@
return {size: 0, value: null};
}
- private decodeUnsignedInt(args: Uint8Array): DecodedArg {
- const arg = this.decodeSignedInt(args);
+ private decodeUnsignedInt(args: Uint8Array, lengthSpecifier: string): DecodedArg {
+ const arg = this._decodeInt(args, true);
+ const bits = ['ll', 'j'].indexOf(lengthSpecifier) !== -1 ? 64 : 32;
// Since ZigZag encoding is used, unsigned integers must be masked off to
// their original bit length.
if (arg.value !== null) {
- let num = arg.value as number;
- num = num >>> 0;
- arg.value = num;
+ let num = arg.value as Long;
+ if (bits === 32) {
+ num = num.and((Long.fromInt(1).shiftLeft(bits)).add(-1));
+ }
+ else {
+ num = num.and(-1);
+ }
+ arg.value = num.toString();
}
return arg;
}
@@ -100,8 +119,8 @@
const arg = this.decodeSignedInt(args);
if (arg.value !== null) {
- const num = arg.value as number;
- arg.value = String.fromCharCode(num);
+ const num = arg.value as Long;
+ arg.value = String.fromCharCode(num.toInt());
}
return arg;
}
@@ -116,7 +135,7 @@
return {size: 4, value: floatValue};
}
- private format(specifierType: string, args: Uint8Array, precision: string): DecodedArg {
+ private format(specifierType: string, args: Uint8Array, precision: string, lengthSpecifier: string): DecodedArg {
if (specifierType == '%') return {size: 0, value: '%'}; // literal %
if (specifierType === 's') {
return this.decodeString(args);
@@ -128,7 +147,7 @@
return this.decodeSignedInt(args);
}
if (UNSIGNED_INT.indexOf(specifierType) !== -1) {
- return this.decodeUnsignedInt(args);
+ return this.decodeUnsignedInt(args, lengthSpecifier);
}
if (FLOATING_POINT.indexOf(specifierType) !== -1) {
return this.decodeFloat(args, precision);
@@ -141,11 +160,11 @@
decode(formatString: string, args: Uint8Array): string {
return formatString.replace(
SPECIFIER_REGEX,
- (_specifier, _precisionFull, precision, specifierType) => {
- const decodedArg = this.format(specifierType, args, precision);
- args = args.slice(decodedArg.size);
- if (decodedArg === null) return '';
- return String(decodedArg.value);
- });
+ (_specifier, _precisionFull, precision, lengthSpecifier, specifierType) => {
+ const decodedArg = this.format(specifierType, args, precision, lengthSpecifier);
+ args = args.slice(decodedArg.size);
+ if (decodedArg === null) return '';
+ return String(decodedArg.value);
+ });
}
}
diff --git a/pw_tokenizer/ts/printf_decoder_test.ts b/pw_tokenizer/ts/printf_decoder_test.ts
index 80814ff..db28e48 100644
--- a/pw_tokenizer/ts/printf_decoder_test.ts
+++ b/pw_tokenizer/ts/printf_decoder_test.ts
@@ -14,6 +14,7 @@
/* eslint-env browser */
import {PrintfDecoder} from './printf_decoder';
+import IntDB from './int_testdata';
function argFromString(arg: string): Uint8Array {
const data = new TextEncoder().encode(arg);
@@ -52,6 +53,41 @@
).toEqual('Hello Mac and PC');
});
+ it('formats string + number correctly', () => {
+ expect(
+ printfDecoder.decode(
+ 'Hello %s and %u',
+ argsConcat(argFromString('Computer'), argFromStringBinary('\xff\xff\x03'))
+ )).toEqual(
+ 'Hello Computer and 4294934528');
+ });
+
+ it('formats integers correctly', () => {
+ for (let index = 0; index < IntDB.length; index++) {
+ const testcase = IntDB[index];
+ // Test signed
+ expect(
+ printfDecoder
+ .decode(testcase[0], argFromStringBinary(testcase[4])))
+ .toEqual(testcase[1]);
+
+ // Test unsigned
+ expect(
+ printfDecoder
+ .decode(testcase[2], argFromStringBinary(testcase[4])))
+ .toEqual(testcase[3]);
+ }
+ });
+
+ it('formats string correctly', () => {
+ expect(
+ printfDecoder.decode(
+ 'Hello %s and %s',
+ argsConcat(argFromString('Mac'), argFromString('PC'))
+ )
+ ).toEqual('Hello Mac and PC');
+ });
+
it('formats varint correctly', () => {
const arg = argFromStringBinary('\xff\xff\x03');
expect(printfDecoder.decode('Number %d', arg)).toEqual('Number -32768');
